gluon.fileutils

1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 4 """ 5 | This file is part of the web2py Web Framework 6 | Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu> 7 | License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) 8 9 File operations 10 --------------- 11 """ 12 13 import storage 14 import os 15 import re 16 import tarfile 17 import glob 18 import time 19 import datetime 20 import logging 21 from http import HTTP 22 from gzip import open as gzopen 23 24 25 __all__ = [ 26 'parse_version', 27 'read_file', 28 'write_file', 29 'readlines_file', 30 'up', 31 'abspath', 32 'mktree', 33 'listdir', 34 'recursive_unlink', 35 'cleanpath', 36 'tar', 37 'untar', 38 'tar_compiled', 39 'get_session', 40 'check_credentials', 41 'w2p_pack', 42 'w2p_unpack', 43 'w2p_pack_plugin', 44 'w2p_unpack_plugin', 45 'fix_newlines', 46 'make_fake_file_like_object', 47 ] 48 49

50 -def parse_semantic(version="Version 1.99.0-rc.1+timestamp.2011.09.19.08.23.26"):

51 """Parses a version string according to http://semver.org/ rules 52 53 Args: 54 version(str): the SemVer string 55 56 Returns: 57 tuple: Major, Minor, Patch, Release, Build Date 58 59 """ 60 re_version = re.compile('(\d+)\.(\d+)\.(\d+)(\-(?P<pre>[^\s+]*))?(\+(?P<build>\S*))') 61 m = re_version.match(version.strip().split()[-1]) 62 if not m: 63 return None 64 a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3)) 65 pre_release = m.group('pre') or '' 66 build = m.group('build') or '' 67 if build.startswith('timestamp'): 68 build = datetime.datetime.strptime(build.split('.',1)[1], '%Y.%m.%d.%H.%M.%S') 69 return (a, b, c, pre_release, build)

70

71 -def parse_legacy(version="Version 1.99.0 (2011-09-19 08:23:26)"):

72 """Parses "legacy" version string 73 74 Args: 75 version(str): the version string 76 77 Returns: 78 tuple: Major, Minor, Patch, Release, Build Date 79 80 """ 81 re_version = re.compile('[^\d]+ (\d+)\.(\d+)\.(\d+)\s*$(?P<datetime>.+?)$\s*(?P<type>[a-z]+)?') 82 m = re_version.match(version) 83 a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3)), 84 pre_release = m.group('type') or 'dev' 85 build = datetime.datetime.strptime(m.group('datetime'), '%Y-%m-%d %H:%M:%S') 86 return (a, b, c, pre_release, build)

87

88 -def parse_version(version):

89 """Attempts to parse SemVer, fallbacks on legacy 90 """ 91 version_tuple = parse_semantic(version) 92 if not version_tuple: 93 version_tuple = parse_legacy(version) 94 return version_tuple

95

96 -def read_file(filename, mode='r'):

97 """Returns content from filename, making sure to close the file explicitly 98 on exit. 99 """ 100 f = open(filename, mode) 101 try: 102 return f.read() 103 finally: 104 f.close()

105 106

107 -def write_file(filename, value, mode='w'):

108 """Writes <value> to filename, making sure to close the file 109 explicitly on exit. 110 """ 111 f = open(filename, mode) 112 try: 113 return f.write(value) 114 finally: 115 f.close()

116 117

118 -def readlines_file(filename, mode='r'):

119 """Applies .split('\n') to the output of `read_file()` 120 """ 121 return read_file(filename, mode).split('\n')

122 123

124 -def mktree(path):

125 head, tail = os.path.split(path) 126 if head: 127 if tail: 128 mktree(head) 129 if not os.path.exists(head): 130 os.mkdir(head)

131 132

133 -def listdir( 134 path, 135 expression='^.+$', 136 drop=True, 137 add_dirs=False, 138 sort=True, 139 maxnum = None, 140 ):

141 """ 142 Like `os.listdir()` but you can specify a regex pattern to filter files. 143 If `add_dirs` is True, the returned items will have the full path. 144 """ 145 if path[-1:] != os.path.sep: 146 path = path + os.path.sep 147 if drop: 148 n = len(path) 149 else: 150 n = 0 151 regex = re.compile(expression) 152 items = [] 153 for (root, dirs, files) in os.walk(path, topdown=True): 154 for dir in dirs[:]: 155 if dir.startswith('.'): 156 dirs.remove(dir) 157 if add_dirs: 158 items.append(root[n:]) 159 for file in sorted(files): 160 if regex.match(file) and not file.startswith('.'): 161 items.append(os.path.join(root, file)[n:]) 162 if maxnum and len(items)>=maxnum: 163 break 164 if sort: 165 return sorted(items) 166 else: 167 return items

168 169

170 -def recursive_unlink(f):

171 """Deletes `f`. If it's a folder, also its contents will be deleted 172 """ 173 if os.path.isdir(f): 174 for s in os.listdir(f): 175 recursive_unlink(os.path.join(f, s)) 176 os.rmdir(f) 177 elif os.path.isfile(f): 178 os.unlink(f)

179 180

181 -def cleanpath(path):

182 """Turns any expression/path into a valid filename. replaces / with _ and 183 removes special characters. 184 """ 185 186 items = path.split('.') 187 if len(items) > 1: 188 path = re.sub('[^\w\.]+', '_', '_'.join(items[:-1]) + '.' 189 + ''.join(items[-1:])) 190 else: 191 path = re.sub('[^\w\.]+', '_', ''.join(items[-1:])) 192 return path

193 194

195 -def _extractall(filename, path='.', members=None):

196 # FIXME: this should be dropped because python 2.4 support was dropped 197 if not hasattr(tarfile.TarFile, 'extractall'): 198 from tarfile import ExtractError 199 200 class TarFile(tarfile.TarFile): 201 202 def extractall(self, path='.', members=None): 203 """Extract all members from the archive to the current working 204 directory and set owner, modification time and permissions on 205 directories afterwards. `path' specifies a different directory 206 to extract to. `members' is optional and must be a subset of the 207 list returned by getmembers(). 208 """ 209 210 directories = [] 211 if members is None: 212 members = self 213 for tarinfo in members: 214 if tarinfo.isdir(): 215 216 # Extract directory with a safe mode, so that 217 # all files below can be extracted as well. 218 219 try: 220 os.makedirs(os.path.join(path, 221 tarinfo.name), 0777) 222 except EnvironmentError: 223 pass 224 directories.append(tarinfo) 225 else: 226 self.extract(tarinfo, path) 227 228 # Reverse sort directories. 229 230 directories.sort(lambda a, b: cmp(a.name, b.name)) 231 directories.reverse() 232 233 # Set correct owner, mtime and filemode on directories. 234 235 for tarinfo in directories: 236 path = os.path.join(path, tarinfo.name) 237 try: 238 self.chown(tarinfo, path) 239 self.utime(tarinfo, path) 240 self.chmod(tarinfo, path) 241 except ExtractError, e: 242 if self.errorlevel > 1: 243 raise 244 else: 245 self._dbg(1, 'tarfile: %s' % e)

246 247 _cls = TarFile 248 else: 249 _cls = tarfile.TarFile 250 251 tar = _cls(filename, 'r') 252 ret = tar.extractall(path, members) 253 tar.close() 254 return ret 255 256

257 -def tar(file, dir, expression='^.+$', filenames=None):

258 """Tars dir into file, only tars file that match expression 259 """ 260 261 tar = tarfile.TarFile(file, 'w') 262 try: 263 if filenames is None: 264 filenames = listdir(dir, expression, add_dirs=True) 265 for file in filenames: 266 tar.add(os.path.join(dir, file), file, False) 267 finally: 268 tar.close()

269

270 -def untar(file, dir):

271 """Untar file into dir 272 """ 273 274 _extractall(file, dir)

275 276

277 -def w2p_pack(filename, path, compiled=False, filenames=None):

278 """Packs a web2py application. 279 280 Args: 281 filename(str): path to the resulting archive 282 path(str): path to the application 283 compiled(bool): if `True` packs the compiled version 284 filenames(list): adds filenames to the archive 285 """ 286 filename = abspath(filename) 287 path = abspath(path) 288 tarname = filename + '.tar' 289 if compiled: 290 tar_compiled(tarname, path, '^[\w\.\-]+$') 291 else: 292 tar(tarname, path, '^[\w\.\-]+$', filenames=filenames) 293 w2pfp = gzopen(filename, 'wb') 294 tarfp = open(tarname, 'rb') 295 w2pfp.write(tarfp.read()) 296 w2pfp.close() 297 tarfp.close() 298 os.unlink(tarname)

299

300 -def create_welcome_w2p():

301 if not os.path.exists('welcome.w2p') or os.path.exists('NEWINSTALL'): 302 try: 303 w2p_pack('welcome.w2p', 'applications/welcome') 304 os.unlink('NEWINSTALL') 305 logging.info("New installation: created welcome.w2p file") 306 except: 307 logging.error("New installation error: unable to create welcome.w2p file")

308 309

310 -def w2p_unpack(filename, path, delete_tar=True):

311 312 if filename=='welcome.w2p': 313 create_welcome_w2p() 314 filename = abspath(filename) 315 path = abspath(path) 316 if filename[-4:] == '.w2p' or filename[-3:] == '.gz': 317 if filename[-4:] == '.w2p': 318 tarname = filename[:-4] + '.tar' 319 else: 320 tarname = filename[:-3] + '.tar' 321 fgzipped = gzopen(filename, 'rb') 322 tarfile = open(tarname, 'wb') 323 tarfile.write(fgzipped.read()) 324 tarfile.close() 325 fgzipped.close() 326 else: 327 tarname = filename 328 untar(tarname, path) 329 if delete_tar: 330 os.unlink(tarname)

331 332

333 -def w2p_pack_plugin(filename, path, plugin_name):

334 """Packs the given plugin into a w2p file. 335 Will match files at:: 336 337 <path>/*/plugin_[name].* 338 <path>/*/plugin_[name]/* 339 340 """ 341 filename = abspath(filename) 342 path = abspath(path) 343 if not filename.endswith('web2py.plugin.%s.w2p' % plugin_name): 344 raise Exception("Not a web2py plugin name") 345 plugin_tarball = tarfile.open(filename, 'w:gz') 346 try: 347 app_dir = path 348 while app_dir[-1] == '/': 349 app_dir = app_dir[:-1] 350 files1 = glob.glob( 351 os.path.join(app_dir, '*/plugin_%s.*' % plugin_name)) 352 files2 = glob.glob( 353 os.path.join(app_dir, '*/plugin_%s/*' % plugin_name)) 354 for file in files1 + files2: 355 plugin_tarball.add(file, arcname=file[len(app_dir) + 1:]) 356 finally: 357 plugin_tarball.close()

358 359

360 -def w2p_unpack_plugin(filename, path, delete_tar=True):

361 filename = abspath(filename) 362 path = abspath(path) 363 if not os.path.basename(filename).startswith('web2py.plugin.'): 364 raise Exception("Not a web2py plugin") 365 w2p_unpack(filename, path, delete_tar)

366 367

368 -def tar_compiled(file, dir, expression='^.+$'):

369 """Used to tar a compiled application. 370 The content of models, views, controllers is not stored in the tar file. 371 """ 372 373 tar = tarfile.TarFile(file, 'w') 374 for file in listdir(dir, expression, add_dirs=True): 375 filename = os.path.join(dir, file) 376 if os.path.islink(filename): 377 continue 378 if os.path.isfile(filename) and file[-4:] != '.pyc': 379 if file[:6] == 'models': 380 continue 381 if file[:5] == 'views': 382 continue 383 if file[:11] == 'controllers': 384 continue 385 if file[:7] == 'modules': 386 continue 387 tar.add(filename, file, False) 388 tar.close()

389 390

391 -def up(path):

392 return os.path.dirname(os.path.normpath(path))

393 394

395 -def get_session(request, other_application='admin'):

396 """Checks that user is authorized to access other_application""" 397 if request.application == other_application: 398 raise KeyError 399 try: 400 session_id = request.cookies['session_id_' + other_application].value 401 session_filename = os.path.join( 402 up(request.folder), other_application, 'sessions', session_id) 403 osession = storage.load_storage(session_filename) 404 except Exception, e: 405 osession = storage.Storage() 406 return osession

407

408 -def set_session(request, session, other_application='admin'):

409 """Checks that user is authorized to access other_application""" 410 if request.application == other_application: 411 raise KeyError 412 session_id = request.cookies['session_id_' + other_application].value 413 session_filename = os.path.join( 414 up(request.folder), other_application, 'sessions', session_id) 415 storage.save_storage(session,session_filename)

416

417 -def check_credentials(request, other_application='admin', 418 expiration=60 * 60, gae_login=True):

419 """Checks that user is authorized to access other_application""" 420 if request.env.web2py_runtime_gae: 421 from google.appengine.api import users 422 if users.is_current_user_admin(): 423 return True 424 elif gae_login: 425 login_html = '<a href="%s">Sign in with your google account</a>.' \ 426 % users.create_login_url(request.env.path_info) 427 raise HTTP(200, '<html><body>%s</body></html>' % login_html) 428 else: 429 return False 430 else: 431 t0 = time.time() 432 dt = t0 - expiration 433 s = get_session(request, other_application) 434 r = (s.authorized and s.last_time and s.last_time > dt) 435 if r: 436 s.last_time = t0 437 set_session(request,s,other_application) 438 return r

439 440

441 -def fix_newlines(path):

442 regex = re.compile(r'''(\r 443 |\r| 444 )''') 445 for filename in listdir(path, '.*\.(py|html)$', drop=False): 446 rdata = read_file(filename, 'rb') 447 wdata = regex.sub('\n', rdata) 448 if wdata != rdata: 449 write_file(filename, wdata, 'wb')

450 451

452 -def copystream( 453 src, 454 dest, 455 size, 456 chunk_size=10 ** 5, 457 ):

458 """ 459 this is here because I think there is a bug in shutil.copyfileobj 460 """ 461 while size > 0: 462 if size < chunk_size: 463 data = src.read(size) 464 else: 465 data = src.read(chunk_size) 466 length = len(data) 467 if length > size: 468 (data, length) = (data[:size], size) 469 size -= length 470 if length == 0: 471 break 472 dest.write(data) 473 if length < chunk_size: 474 break 475 dest.seek(0) 476 return

477 478

479 -def make_fake_file_like_object():

480 class LogFile(object): 481 def write(self, value): 482 pass

483 484 def close(self): 485 pass 486 return LogFile() 487 488 489 from settings import global_settings # we need to import settings here because 490 # settings imports fileutils too 491 492

493 -def abspath(*relpath, **base):

494 """Converts relative path to absolute path based (by default) on 495 applications_parent 496 """ 497 path = os.path.join(*relpath) 498 gluon = base.get('gluon', False) 499 if os.path.isabs(path): 500 return path 501 if gluon: 502 return os.path.join(global_settings.gluon_parent, path) 503 return os.path.join(global_settings.applications_parent, path)

504

Source Code for Module gluon.fileutils