1
2
3
4 """
5 | This file is part of the web2py Web Framework
6 | Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
7 | License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
8
9 Functions required to execute app components
10 ---------------------------------------------
11
12 Note:
13 FOR INTERNAL USE ONLY
14 """
15
16 import re
17 import fnmatch
18 import os
19 import copy
20 import random
21 import __builtin__
22 from gluon.storage import Storage, List
23 from gluon.template import parse_template
24 from gluon.restricted import restricted, compile2
25 from gluon.fileutils import mktree, listdir, read_file, write_file
26 from gluon.myregex import regex_expose, regex_longcomments
27 from gluon.languages import translator
28 from gluon.dal import BaseAdapter, SQLDB, SQLField, DAL, Field
29 from gluon.sqlhtml import SQLFORM, SQLTABLE
30 from gluon.cache import Cache
31 from gluon.globals import current, Response
32 from gluon import settings
33 from gluon.cfs import getcfs
34 from gluon import html
35 from gluon import validators
36 from gluon.http import HTTP, redirect
37 import marshal
38 import shutil
39 import imp
40 import logging
41 logger = logging.getLogger("web2py")
42 from gluon import rewrite
43 from custom_import import custom_import_install
44
45 try:
46 import py_compile
47 except:
48 logger.warning('unable to import py_compile')
49
50 is_pypy = settings.global_settings.is_pypy
51 is_gae = settings.global_settings.web2py_runtime_gae
52 is_jython = settings.global_settings.is_jython
53
54 pjoin = os.path.join
55
56 TEST_CODE = \
57 r"""
58 def _TEST():
59 import doctest, sys, cStringIO, types, cgi, gluon.fileutils
60 if not gluon.fileutils.check_credentials(request):
61 raise HTTP(401, web2py_error='invalid credentials')
62 stdout = sys.stdout
63 html = '<h2>Testing controller "%s.py" ... done.</h2><br/>\n' \
64 % request.controller
65 for key in sorted([key for key in globals() if not key in __symbols__+['_TEST']]):
66 eval_key = eval(key)
67 if type(eval_key) == types.FunctionType:
68 number_doctests = sum([len(ds.examples) for ds in doctest.DocTestFinder().find(eval_key)])
69 if number_doctests>0:
70 sys.stdout = cStringIO.StringIO()
71 name = '%s/controllers/%s.py in %s.__doc__' \
72 % (request.folder, request.controller, key)
73 doctest.run_docstring_examples(eval_key,
74 globals(), False, name=name)
75 report = sys.stdout.getvalue().strip()
76 if report:
77 pf = 'failed'
78 else:
79 pf = 'passed'
80 html += '<h3 class="%s">Function %s [%s]</h3>\n' \
81 % (pf, key, pf)
82 if report:
83 html += CODE(report, language='web2py', \
84 link='/examples/global/vars/').xml()
85 html += '<br/>\n'
86 else:
87 html += \
88 '<h3 class="nodoctests">Function %s [no doctests]</h3><br/>\n' \
89 % (key)
90 response._vars = html
91 sys.stdout = stdout
92 _TEST()
93 """
94
95 CACHED_REGEXES = {}
96 CACHED_REGEXES_MAX_SIZE = 1000
97
98
107
108
110 """
111 NOTE could simple use a dict and populate it,
112 NOTE not sure if this changes things though if monkey patching import.....
113 """
114
116 try:
117 return getattr(__builtin__, key)
118 except AttributeError:
119 raise KeyError(key)
120
122 setattr(self, key, value)
123
124
125 -def LOAD(c=None, f='index', args=None, vars=None,
126 extension=None, target=None, ajax=False, ajax_trap=False,
127 url=None, user_signature=False, timeout=None, times=1,
128 content='loading...', **attr):
129 """ LOADs a component into the action's document
130
131 Args:
132 c(str): controller
133 f(str): function
134 args(tuple or list): arguments
135 vars(dict): vars
136 extension(str): extension
137 target(str): id of the target
138 ajax(bool): True to enable AJAX bahaviour
139 ajax_trap(bool): True if `ajax` is set to `True`, traps
140 both links and forms "inside" the target
141 url(str): overrides `c`,`f`,`args` and `vars`
142 user_signature(bool): adds hmac signature to all links
143 with a key that is different for every user
144 timeout(int): in milliseconds, specifies the time to wait before
145 starting the request or the frequency if times is greater than
146 1 or "infinity"
147 times(integer or str): how many times the component will be requested
148 "infinity" or "continuous" are accepted to reload indefinitely the
149 component
150 """
151 from html import TAG, DIV, URL, SCRIPT, XML
152 if args is None:
153 args = []
154 vars = Storage(vars or {})
155 target = target or 'c' + str(random.random())[2:]
156 attr['_id'] = target
157 request = current.request
158 if '.' in f:
159 f, extension = f.rsplit('.', 1)
160 if url or ajax:
161 url = url or URL(request.application, c, f, r=request,
162 args=args, vars=vars, extension=extension,
163 user_signature=user_signature)
164
165 if isinstance(times, basestring):
166 if times.upper() in ("INFINITY", "CONTINUOUS"):
167 times = "Infinity"
168 else:
169 raise TypeError("Unsupported times argument %s" % times)
170 elif isinstance(times, int):
171 if times <= 0:
172 raise ValueError("Times argument must be greater than zero, 'Infinity' or None")
173 else:
174 raise TypeError("Unsupported times argument type %s" % type(times))
175 if timeout is not None:
176 if not isinstance(timeout, (int, long)):
177 raise ValueError("Timeout argument must be an integer or None")
178 elif timeout <= 0:
179 raise ValueError(
180 "Timeout argument must be greater than zero or None")
181 statement = "$.web2py.component('%s','%s', %s, %s);" \
182 % (url, target, timeout, times)
183 attr['_data-w2p_timeout'] = timeout
184 attr['_data-w2p_times'] = times
185 else:
186 statement = "$.web2py.component('%s','%s');" % (url, target)
187 attr['_data-w2p_remote'] = url
188 if not target is None:
189 return DIV(content, **attr)
190
191 else:
192 if not isinstance(args, (list, tuple)):
193 args = [args]
194 c = c or request.controller
195 other_request = Storage(request)
196 other_request['env'] = Storage(request.env)
197 other_request.controller = c
198 other_request.function = f
199 other_request.extension = extension or request.extension
200 other_request.args = List(args)
201 other_request.vars = vars
202 other_request.get_vars = vars
203 other_request.post_vars = Storage()
204 other_response = Response()
205 other_request.env.path_info = '/' + \
206 '/'.join([request.application, c, f] +
207 map(str, other_request.args))
208 other_request.env.query_string = \
209 vars and URL(vars=vars).split('?')[1] or ''
210 other_request.env.http_web2py_component_location = \
211 request.env.path_info
212 other_request.cid = target
213 other_request.env.http_web2py_component_element = target
214 other_response.view = '%s/%s.%s' % (c, f, other_request.extension)
215
216 other_environment = copy.copy(current.globalenv)
217
218 other_response._view_environment = other_environment
219 other_response.generic_patterns = \
220 copy.copy(current.response.generic_patterns)
221 other_environment['request'] = other_request
222 other_environment['response'] = other_response
223
224
225
226 original_request, current.request = current.request, other_request
227 original_response, current.response = current.response, other_response
228 page = run_controller_in(c, f, other_environment)
229 if isinstance(page, dict):
230 other_response._vars = page
231 other_response._view_environment.update(page)
232 run_view_in(other_response._view_environment)
233 page = other_response.body.getvalue()
234 current.request, current.response = original_request, original_response
235 js = None
236 if ajax_trap:
237 link = URL(request.application, c, f, r=request,
238 args=args, vars=vars, extension=extension,
239 user_signature=user_signature)
240 js = "$.web2py.trap_form('%s','%s');" % (link, target)
241 script = js and SCRIPT(js, _type="text/javascript") or ''
242 return TAG[''](DIV(XML(page), **attr), script)
243
244
246 """
247 Attention: this helper is new and experimental
248 """
250 self.environment = environment
251
252 - def __call__(self, c=None, f='index', args=None, vars=None,
253 extension=None, target=None, ajax=False, ajax_trap=False,
254 url=None, user_signature=False, content='loading...', **attr):
255 if args is None:
256 args = []
257 vars = Storage(vars or {})
258 import globals
259 target = target or 'c' + str(random.random())[2:]
260 attr['_id'] = target
261 request = self.environment['request']
262 if '.' in f:
263 f, extension = f.rsplit('.', 1)
264 if url or ajax:
265 url = url or html.URL(request.application, c, f, r=request,
266 args=args, vars=vars, extension=extension,
267 user_signature=user_signature)
268 script = html.SCRIPT('$.web2py.component("%s","%s")' % (url, target),
269 _type="text/javascript")
270 return html.TAG[''](script, html.DIV(content, **attr))
271 else:
272 if not isinstance(args, (list, tuple)):
273 args = [args]
274 c = c or request.controller
275
276 other_request = Storage(request)
277 other_request['env'] = Storage(request.env)
278 other_request.controller = c
279 other_request.function = f
280 other_request.extension = extension or request.extension
281 other_request.args = List(args)
282 other_request.vars = vars
283 other_request.get_vars = vars
284 other_request.post_vars = Storage()
285 other_response = globals.Response()
286 other_request.env.path_info = '/' + \
287 '/'.join([request.application, c, f] +
288 map(str, other_request.args))
289 other_request.env.query_string = \
290 vars and html.URL(vars=vars).split('?')[1] or ''
291 other_request.env.http_web2py_component_location = \
292 request.env.path_info
293 other_request.cid = target
294 other_request.env.http_web2py_component_element = target
295 other_response.view = '%s/%s.%s' % (c, f, other_request.extension)
296 other_environment = copy.copy(self.environment)
297 other_response._view_environment = other_environment
298 other_response.generic_patterns = \
299 copy.copy(current.response.generic_patterns)
300 other_environment['request'] = other_request
301 other_environment['response'] = other_response
302
303
304
305 original_request, current.request = current.request, other_request
306 original_response, current.response = current.response, other_response
307 page = run_controller_in(c, f, other_environment)
308 if isinstance(page, dict):
309 other_response._vars = page
310 other_response._view_environment.update(page)
311 run_view_in(other_response._view_environment)
312 page = other_response.body.getvalue()
313 current.request, current.response = original_request, original_response
314 js = None
315 if ajax_trap:
316 link = html.URL(request.application, c, f, r=request,
317 args=args, vars=vars, extension=extension,
318 user_signature=user_signature)
319 js = "$.web2py.trap_form('%s','%s');" % (link, target)
320 script = js and html.SCRIPT(js, _type="text/javascript") or ''
321 return html.TAG[''](html.DIV(html.XML(page), **attr), script)
322
323
325 """
326 In apps, instead of importing a local module
327 (in applications/app/modules) with::
328
329 import a.b.c as d
330
331 you should do::
332
333 d = local_import('a.b.c')
334
335 or (to force a reload):
336
337 d = local_import('a.b.c', reload=True)
338
339 This prevents conflict between applications and un-necessary execs.
340 It can be used to import any module, including regular Python modules.
341 """
342 items = name.replace('/', '.')
343 name = "applications.%s.modules.%s" % (app, items)
344 module = __import__(name)
345 for item in name.split(".")[1:]:
346 module = getattr(module, item)
347 if reload_force:
348 reload(module)
349 return module
350
351
352 """
353 OLD IMPLEMENTATION:
354 items = name.replace('/','.').split('.')
355 filename, modulepath = items[-1], pjoin(apath,'modules',*items[:-1])
356 imp.acquire_lock()
357 try:
358 file=None
359 (file,path,desc) = imp.find_module(filename,[modulepath]+sys.path)
360 if not path in sys.modules or reload:
361 if is_gae:
362 module={}
363 execfile(path,{},module)
364 module=Storage(module)
365 else:
366 module = imp.load_module(path,file,path,desc)
367 sys.modules[path] = module
368 else:
369 module = sys.modules[path]
370 except Exception, e:
371 module = None
372 if file:
373 file.close()
374 imp.release_lock()
375 if not module:
376 raise ImportError, "cannot find module %s in %s" % (
377 filename, modulepath)
378 return module
379 """
380
381 _base_environment_ = dict((k, getattr(html, k)) for k in html.__all__)
382 _base_environment_.update(
383 (k, getattr(validators, k)) for k in validators.__all__)
384 _base_environment_['__builtins__'] = __builtins__
385 _base_environment_['HTTP'] = HTTP
386 _base_environment_['redirect'] = redirect
387 _base_environment_['DAL'] = DAL
388 _base_environment_['Field'] = Field
389 _base_environment_['SQLDB'] = SQLDB
390 _base_environment_['SQLField'] = SQLField
391 _base_environment_['SQLFORM'] = SQLFORM
392 _base_environment_['SQLTABLE'] = SQLTABLE
393 _base_environment_['LOAD'] = LOAD
394
396 """
397 Build the environment dictionary into which web2py files are executed.
398 """
399
400 environment = dict(_base_environment_)
401
402 if not request.env:
403 request.env = Storage()
404
405
406 response.models_to_run = [
407 r'^\w+\.py$',
408 r'^%s/\w+\.py$' % request.controller,
409 r'^%s/%s/\w+\.py$' % (request.controller, request.function)
410 ]
411
412 t = environment['T'] = translator(os.path.join(request.folder,'languages'),
413 request.env.http_accept_language)
414 c = environment['cache'] = Cache(request)
415
416 if store_current:
417 current.globalenv = environment
418 current.request = request
419 current.response = response
420 current.session = session
421 current.T = t
422 current.cache = c
423
424 global __builtins__
425 if is_jython:
426 __builtins__ = mybuiltin()
427 elif is_pypy:
428 __builtins__ = mybuiltin()
429 else:
430 __builtins__['__import__'] = __builtin__.__import__
431 environment['request'] = request
432 environment['response'] = response
433 environment['session'] = session
434 environment['local_import'] = \
435 lambda name, reload=False, app=request.application:\
436 local_import_aux(name, reload, app)
437 BaseAdapter.set_folder(pjoin(request.folder, 'databases'))
438 response._view_environment = copy.copy(environment)
439 custom_import_install()
440 return environment
441
442
444 """
445 Bytecode compiles the file `filename`
446 """
447 py_compile.compile(filename)
448
449
451 """
452 Read the code inside a bytecode compiled file if the MAGIC number is
453 compatible
454
455 Returns:
456 a code object
457 """
458 data = read_file(filename, 'rb')
459 if not is_gae and data[:4] != imp.get_magic():
460 raise SystemError('compiled code is incompatible')
461 return marshal.loads(data[8:])
462
463
465 """
466 Compiles all the views in the application specified by `folder`
467 """
468
469 path = pjoin(folder, 'views')
470 for fname in listdir(path, '^[\w/\-]+(\.\w+)*$'):
471 try:
472 data = parse_template(fname, path)
473 except Exception, e:
474 raise Exception("%s in %s" % (e, fname))
475 filename = 'views.%s.py' % fname.replace(os.path.sep, '.')
476 filename = pjoin(folder, 'compiled', filename)
477 write_file(filename, data)
478 save_pyc(filename)
479 os.unlink(filename)
480
481
483 """
484 Compiles all the models in the application specified by `folder`
485 """
486
487 path = pjoin(folder, 'models')
488 for fname in listdir(path, '.+\.py$'):
489 data = read_file(pjoin(path, fname))
490 modelfile = 'models.'+fname.replace(os.path.sep,'.')
491 filename = pjoin(folder, 'compiled', modelfile)
492 mktree(filename)
493 write_file(filename, data)
494 save_pyc(filename)
495 os.unlink(filename)
496
500
502 """
503 Compiles all the controllers in the application specified by `folder`
504 """
505
506 path = pjoin(folder, 'controllers')
507 for fname in listdir(path, '.+\.py$'):
508
509 data = read_file(pjoin(path, fname))
510 exposed = find_exposed_functions(data)
511 for function in exposed:
512 command = data + "\nresponse._vars=response._caller(%s)\n" % \
513 function
514 filename = pjoin(folder, 'compiled',
515 'controllers.%s.%s.py' % (fname[:-3],function))
516 write_file(filename, command)
517 save_pyc(filename)
518 os.unlink(filename)
519
522
525
527 """
528 Runs all models (in the app specified by the current folder)
529 It tries pre-compiled models first before compiling them.
530 """
531
532 folder = environment['request'].folder
533 c = environment['request'].controller
534 f = environment['request'].function
535 response = environment['response']
536
537 path = pjoin(folder, 'models')
538 cpath = pjoin(folder, 'compiled')
539 compiled = os.path.exists(cpath)
540 if compiled:
541 models = sorted(listdir(cpath, '^models[_.][\w.]+\.pyc$', 0), model_cmp)
542 else:
543 models = sorted(listdir(path, '^\w+\.py$', 0, sort=False), model_cmp_sep)
544 models_to_run = None
545 for model in models:
546 if response.models_to_run != models_to_run:
547 regex = models_to_run = response.models_to_run[:]
548 if isinstance(regex, list):
549 regex = re_compile('|'.join(regex))
550 if models_to_run:
551 if compiled:
552 n = len(cpath)+8
553 fname = model[n:-4].replace('.','/')+'.py'
554 else:
555 n = len(path)+1
556 fname = model[n:].replace(os.path.sep,'/')
557 if not regex.search(fname) and c != 'appadmin':
558 continue
559 elif compiled:
560 code = read_pyc(model)
561 elif is_gae:
562 code = getcfs(model, model,
563 lambda: compile2(read_file(model), model))
564 else:
565 code = getcfs(model, model, None)
566 restricted(code, environment, layer=model)
567
568
570 """
571 Runs the controller.function() (for the app specified by
572 the current folder).
573 It tries pre-compiled controller_function.pyc first before compiling it.
574 """
575
576
577 folder = environment['request'].folder
578 path = pjoin(folder, 'compiled')
579 badc = 'invalid controller (%s/%s)' % (controller, function)
580 badf = 'invalid function (%s/%s)' % (controller, function)
581 if os.path.exists(path):
582 filename = pjoin(path, 'controllers.%s.%s.pyc'
583 % (controller, function))
584 if not os.path.exists(filename):
585
586 filename = pjoin(path, 'controllers_%s_%s.pyc'
587 % (controller, function))
588
589 if not os.path.exists(filename):
590 raise HTTP(404,
591 rewrite.THREAD_LOCAL.routes.error_message % badf,
592 web2py_error=badf)
593 restricted(read_pyc(filename), environment, layer=filename)
594 elif function == '_TEST':
595
596 from settings import global_settings
597 from admin import abspath, add_path_first
598 paths = (global_settings.gluon_parent, abspath(
599 'site-packages', gluon=True), abspath('gluon', gluon=True), '')
600 [add_path_first(path) for path in paths]
601
602
603 filename = pjoin(folder, 'controllers/%s.py'
604 % controller)
605 if not os.path.exists(filename):
606 raise HTTP(404,
607 rewrite.THREAD_LOCAL.routes.error_message % badc,
608 web2py_error=badc)
609 environment['__symbols__'] = environment.keys()
610 code = read_file(filename)
611 code += TEST_CODE
612 restricted(code, environment, layer=filename)
613 else:
614 filename = pjoin(folder, 'controllers/%s.py'
615 % controller)
616 if not os.path.exists(filename):
617 raise HTTP(404,
618 rewrite.THREAD_LOCAL.routes.error_message % badc,
619 web2py_error=badc)
620 code = read_file(filename)
621 exposed = find_exposed_functions(code)
622 if not function in exposed:
623 raise HTTP(404,
624 rewrite.THREAD_LOCAL.routes.error_message % badf,
625 web2py_error=badf)
626 code = "%s\nresponse._vars=response._caller(%s)\n" % (code, function)
627 if is_gae:
628 layer = filename + ':' + function
629 code = getcfs(layer, filename, lambda: compile2(code, layer))
630 restricted(code, environment, filename)
631 response = environment['response']
632 vars = response._vars
633 if response.postprocessing:
634 vars = reduce(lambda vars, p: p(vars), response.postprocessing, vars)
635 if isinstance(vars, unicode):
636 vars = vars.encode('utf8')
637 elif hasattr(vars, 'xml') and callable(vars.xml):
638 vars = vars.xml()
639 return vars
640
641
643 """
644 Executes the view for the requested action.
645 The view is the one specified in `response.view` or determined by the url
646 or `view/generic.extension`
647 It tries the pre-compiled views_controller_function.pyc before compiling it.
648 """
649 request = environment['request']
650 response = environment['response']
651 view = response.view
652 folder = request.folder
653 path = pjoin(folder, 'compiled')
654 badv = 'invalid view (%s)' % view
655 if response.generic_patterns:
656 patterns = response.generic_patterns
657 regex = re_compile('|'.join(map(fnmatch.translate, patterns)))
658 short_action = '%(controller)s/%(function)s.%(extension)s' % request
659 allow_generic = regex.search(short_action)
660 else:
661 allow_generic = False
662 if not isinstance(view, str):
663 ccode = parse_template(view, pjoin(folder, 'views'),
664 context=environment)
665 restricted(ccode, environment, 'file stream')
666 elif os.path.exists(path):
667 x = view.replace('/', '.')
668 files = ['views.%s.pyc' % x]
669 if allow_generic:
670 files.append('views.generic.%s.pyc' % request.extension)
671
672 x = view.replace('/', '_')
673 files.append('views_%s.pyc' % x)
674 if allow_generic:
675 files.append('views_generic.%s.pyc' % request.extension)
676 if request.extension == 'html':
677 files.append('views_%s.pyc' % x[:-5])
678 if allow_generic:
679 files.append('views_generic.pyc')
680
681 for f in files:
682 filename = pjoin(path, f)
683 if os.path.exists(filename):
684 code = read_pyc(filename)
685 restricted(code, environment, layer=filename)
686 return
687 raise HTTP(404,
688 rewrite.THREAD_LOCAL.routes.error_message % badv,
689 web2py_error=badv)
690 else:
691 filename = pjoin(folder, 'views', view)
692 if not os.path.exists(filename) and allow_generic:
693 view = 'generic.' + request.extension
694 filename = pjoin(folder, 'views', view)
695 if not os.path.exists(filename):
696 raise HTTP(404,
697 rewrite.THREAD_LOCAL.routes.error_message % badv,
698 web2py_error=badv)
699 layer = filename
700 if is_gae:
701 ccode = getcfs(layer, filename,
702 lambda: compile2(parse_template(view,
703 pjoin(folder, 'views'),
704 context=environment), layer))
705 else:
706 ccode = parse_template(view,
707 pjoin(folder, 'views'),
708 context=environment)
709 restricted(ccode, environment, layer)
710
711
713 """
714 Deletes the folder `compiled` containing the compiled application.
715 """
716 try:
717 shutil.rmtree(pjoin(folder, 'compiled'))
718 path = pjoin(folder, 'controllers')
719 for file in listdir(path, '.*\.pyc$', drop=False):
720 os.unlink(file)
721 except OSError:
722 pass
723
724
734
735
737 """
738 Example::
739
740 >>> import traceback, types
741 >>> environment={'x':1}
742 >>> open('a.py', 'w').write('print 1/x')
743 >>> save_pyc('a.py')
744 >>> os.unlink('a.py')
745 >>> if type(read_pyc('a.pyc'))==types.CodeType: print 'code'
746 code
747 >>> exec read_pyc('a.pyc') in environment
748 1
749 """
750
751 return
752
753
754 if __name__ == '__main__':
755 import doctest
756 doctest.testmod()
757