gluon.newcron

1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 4 """ 5 | This file is part of the web2py Web Framework 6 | Created by Attila Csipa <web2py@csipa.in.rs> 7 | Modified by Massimo Di Pierro <mdipierro@cs.depaul.edu> 8 | License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) 9 10 Cron-style interface 11 """ 12 13 import sys 14 import os 15 import threading 16 import logging 17 import time 18 import sched 19 import re 20 import datetime 21 import platform 22 import portalocker 23 import fileutils 24 import cPickle 25 from gluon.settings import global_settings 26 27 logger = logging.getLogger("web2py.cron") 28 _cron_stopping = False 29 _cron_subprocs = [] 30 31

32 -def absolute_path_link(path):

33 """ 34 Returns an absolute path for the destination of a symlink 35 36 """ 37 if os.path.islink(path): 38 link = os.readlink(path) 39 if not os.path.isabs(link): 40 link = os.path.join(os.path.dirname(path), link) 41 else: 42 link = os.path.abspath(path) 43 return link

44 45

46 -def stopcron():

47 "Graceful shutdown of cron" 48 global _cron_stopping 49 _cron_stopping = True 50 while _cron_subprocs: 51 _cron_subprocs.pop().terminate()

52

53 -class extcron(threading.Thread):

54

55 - def __init__(self, applications_parent, apps=None):

56 threading.Thread.__init__(self) 57 self.setDaemon(False) 58 self.path = applications_parent 59 self.apps = apps

60 # crondance(self.path, 'external', startup=True, apps=self.apps) 61

62 - def run(self):

63 if not _cron_stopping: 64 logger.debug('external cron invocation') 65 crondance(self.path, 'external', startup=False, apps=self.apps)

66 67

68 -class hardcron(threading.Thread):

69

70 - def __init__(self, applications_parent):

71 threading.Thread.__init__(self) 72 self.setDaemon(True) 73 self.path = applications_parent 74 crondance(self.path, 'hard', startup=True)

75

76 - def launch(self):

77 if not _cron_stopping: 78 logger.debug('hard cron invocation') 79 crondance(self.path, 'hard', startup=False)

80

81 - def run(self):

82 s = sched.scheduler(time.time, time.sleep) 83 logger.info('Hard cron daemon started') 84 while not _cron_stopping: 85 now = time.time() 86 s.enter(60 - now % 60, 1, self.launch, ()) 87 s.run()

88 89

90 -class softcron(threading.Thread):

91

92 - def __init__(self, applications_parent):

93 threading.Thread.__init__(self) 94 self.path = applications_parent

95 # crondance(self.path, 'soft', startup=True) 96

97 - def run(self):

98 if not _cron_stopping: 99 logger.debug('soft cron invocation') 100 crondance(self.path, 'soft', startup=False)

101 102

103 -class Token(object):

104

105 - def __init__(self, path):

106 self.path = os.path.join(path, 'cron.master') 107 if not os.path.exists(self.path): 108 fileutils.write_file(self.path, '', 'wb') 109 self.master = None 110 self.now = time.time()

111

112 - def acquire(self, startup=False):

113 """ 114 Returns the time when the lock is acquired or 115 None if cron already running 116 117 lock is implemented by writing a pickle (start, stop) in cron.master 118 start is time when cron job starts and stop is time when cron completed 119 stop == 0 if job started but did not yet complete 120 if a cron job started within less than 60 seconds, acquire returns None 121 if a cron job started before 60 seconds and did not stop, 122 a warning is issue "Stale cron.master detected" 123 """ 124 if sys.platform == 'win32': 125 locktime = 59.5 126 else: 127 locktime = 59.99 128 if portalocker.LOCK_EX is None: 129 logger.warning('WEB2PY CRON: Disabled because no file locking') 130 return None 131 self.master = open(self.path, 'rb+') 132 try: 133 ret = None 134 portalocker.lock(self.master, portalocker.LOCK_EX) 135 try: 136 (start, stop) = cPickle.load(self.master) 137 except: 138 (start, stop) = (0, 1) 139 if startup or self.now - start > locktime: 140 ret = self.now 141 if not stop: 142 # this happens if previous cron job longer than 1 minute 143 logger.warning('WEB2PY CRON: Stale cron.master detected') 144 logger.debug('WEB2PY CRON: Acquiring lock') 145 self.master.seek(0) 146 cPickle.dump((self.now, 0), self.master) 147 self.master.flush() 148 finally: 149 portalocker.unlock(self.master) 150 if not ret: 151 # do this so no need to release 152 self.master.close() 153 return ret

154

155 - def release(self):

156 """ 157 Writes into cron.master the time when cron job was completed 158 """ 159 if not self.master.closed: 160 portalocker.lock(self.master, portalocker.LOCK_EX) 161 logger.debug('WEB2PY CRON: Releasing cron lock') 162 self.master.seek(0) 163 (start, stop) = cPickle.load(self.master) 164 if start == self.now: # if this is my lock 165 self.master.seek(0) 166 cPickle.dump((self.now, time.time()), self.master) 167 portalocker.unlock(self.master) 168 self.master.close()

169 170

171 -def rangetolist(s, period='min'):

172 retval = [] 173 if s.startswith('*'): 174 if period == 'min': 175 s = s.replace('*', '0-59', 1) 176 elif period == 'hr': 177 s = s.replace('*', '0-23', 1) 178 elif period == 'dom': 179 s = s.replace('*', '1-31', 1) 180 elif period == 'mon': 181 s = s.replace('*', '1-12', 1) 182 elif period == 'dow': 183 s = s.replace('*', '0-6', 1) 184 m = re.compile(r'(\d+)-(\d+)/(\d+)') 185 match = m.match(s) 186 if match: 187 for i in range(int(match.group(1)), int(match.group(2)) + 1): 188 if i % int(match.group(3)) == 0: 189 retval.append(i) 190 return retval

191 192

193 -def parsecronline(line):

194 task = {} 195 if line.startswith('@reboot'): 196 line = line.replace('@reboot', '-1 * * * *') 197 elif line.startswith('@yearly'): 198 line = line.replace('@yearly', '0 0 1 1 *') 199 elif line.startswith('@annually'): 200 line = line.replace('@annually', '0 0 1 1 *') 201 elif line.startswith('@monthly'): 202 line = line.replace('@monthly', '0 0 1 * *') 203 elif line.startswith('@weekly'): 204 line = line.replace('@weekly', '0 0 * * 0') 205 elif line.startswith('@daily'): 206 line = line.replace('@daily', '0 0 * * *') 207 elif line.startswith('@midnight'): 208 line = line.replace('@midnight', '0 0 * * *') 209 elif line.startswith('@hourly'): 210 line = line.replace('@hourly', '0 * * * *') 211 params = line.strip().split(None, 6) 212 if len(params) < 7: 213 return None 214 daysofweek = {'sun': 0, 'mon': 1, 'tue': 2, 'wed': 3, 'thu': 4, 215 'fri': 5, 'sat': 6} 216 for (s, id) in zip(params[:5], ['min', 'hr', 'dom', 'mon', 'dow']): 217 if not s in [None, '*']: 218 task[id] = [] 219 vals = s.split(',') 220 for val in vals: 221 if val != '-1' and '-' in val and '/' not in val: 222 val = '%s/1' % val 223 if '/' in val: 224 task[id] += rangetolist(val, id) 225 elif val.isdigit() or val == '-1': 226 task[id].append(int(val)) 227 elif id == 'dow' and val[:3].lower() in daysofweek: 228 task[id].append(daysofweek(val[:3].lower())) 229 task['user'] = params[5] 230 task['cmd'] = params[6] 231 return task

232 233

234 -class cronlauncher(threading.Thread):

235

236 - def __init__(self, cmd, shell=True):

237 threading.Thread.__init__(self) 238 if platform.system() == 'Windows': 239 shell = False 240 self.cmd = cmd 241 self.shell = shell

242

243 - def run(self):

244 import subprocess 245 global _cron_subprocs 246 if isinstance(self.cmd, (list, tuple)): 247 cmd = self.cmd 248 else: 249 cmd = self.cmd.split() 250 proc = subprocess.Popen(cmd, 251 stdin=subprocess.PIPE, 252 stdout=subprocess.PIPE, 253 stderr=subprocess.PIPE, 254 shell=self.shell) 255 _cron_subprocs.append(proc) 256 (stdoutdata, stderrdata) = proc.communicate() 257 _cron_subprocs.remove(proc) 258 if proc.returncode != 0: 259 logger.warning( 260 'WEB2PY CRON Call returned code %s:\n%s' % 261 (proc.returncode, stdoutdata + stderrdata)) 262 else: 263 logger.debug('WEB2PY CRON Call returned success:\n%s' 264 % stdoutdata)

265 266

267 -def crondance(applications_parent, ctype='soft', startup=False, apps=None):

268 apppath = os.path.join(applications_parent, 'applications') 269 cron_path = os.path.join(applications_parent) 270 token = Token(cron_path) 271 cronmaster = token.acquire(startup=startup) 272 if not cronmaster: 273 return 274 now_s = time.localtime() 275 checks = (('min', now_s.tm_min), 276 ('hr', now_s.tm_hour), 277 ('mon', now_s.tm_mon), 278 ('dom', now_s.tm_mday), 279 ('dow', (now_s.tm_wday + 1) % 7)) 280 281 if apps is None: 282 apps = [x for x in os.listdir(apppath) 283 if os.path.isdir(os.path.join(apppath, x))] 284 285 full_apath_links = set() 286 287 for app in apps: 288 if _cron_stopping: 289 break 290 apath = os.path.join(apppath, app) 291 292 # if app is a symbolic link to other app, skip it 293 full_apath_link = absolute_path_link(apath) 294 if full_apath_link in full_apath_links: 295 continue 296 else: 297 full_apath_links.add(full_apath_link) 298 299 cronpath = os.path.join(apath, 'cron') 300 crontab = os.path.join(cronpath, 'crontab') 301 if not os.path.exists(crontab): 302 continue 303 try: 304 cronlines = fileutils.readlines_file(crontab, 'rt') 305 lines = [x.strip() for x in cronlines if x.strip( 306 ) and not x.strip().startswith('#')] 307 tasks = [parsecronline(cline) for cline in lines] 308 except Exception, e: 309 logger.error('WEB2PY CRON: crontab read error %s' % e) 310 continue 311 312 for task in tasks: 313 if _cron_stopping: 314 break 315 if sys.executable.lower().endswith('pythonservice.exe'): 316 _python_exe = os.path.join(sys.exec_prefix, 'python.exe') 317 else: 318 _python_exe = sys.executable 319 commands = [_python_exe] 320 w2p_path = fileutils.abspath('web2py.py', gluon=True) 321 if os.path.exists(w2p_path): 322 commands.append(w2p_path) 323 if global_settings.applications_parent != global_settings.gluon_parent: 324 commands.extend(('-f', global_settings.applications_parent)) 325 citems = [(k in task and not v in task[k]) for k, v in checks] 326 task_min = task.get('min', []) 327 if not task: 328 continue 329 elif not startup and task_min == [-1]: 330 continue 331 elif task_min != [-1] and reduce(lambda a, b: a or b, citems): 332 continue 333 logger.info('WEB2PY CRON (%s): %s executing %s in %s at %s' 334 % (ctype, app, task.get('cmd'), 335 os.getcwd(), datetime.datetime.now())) 336 action, command, models = False, task['cmd'], '' 337 if command.startswith('**'): 338 (action, models, command) = (True, '', command[2:]) 339 elif command.startswith('*'): 340 (action, models, command) = (True, '-M', command[1:]) 341 else: 342 action = False 343 344 if action and command.endswith('.py'): 345 commands.extend(('-J', # cron job 346 models, # import models? 347 '-S', app, # app name 348 '-a', '"<recycle>"', # password 349 '-R', command)) # command 350 elif action: 351 commands.extend(('-J', # cron job 352 models, # import models? 353 '-S', app + '/' + command, # app name 354 '-a', '"<recycle>"')) # password 355 else: 356 commands = command 357 358 # from python docs: 359 # You do not need shell=True to run a batch file or 360 # console-based executable. 361 shell = False 362 363 try: 364 cronlauncher(commands, shell=shell).start() 365 except Exception, e: 366 logger.warning( 367 'WEB2PY CRON: Execution error for %s: %s' 368 % (task.get('cmd'), e)) 369 token.release()

370

Source Code for Module gluon.newcron