gluon.languages

1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 4 """ 5 | This file is part of the web2py Web Framework 6 | Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu> 7 | License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) 8 | Plural subsystem is created by Vladyslav Kozlovskyy (Ukraine) <dbdevelop@gmail.com> 9 10 Translation system 11 -------------------------------------------- 12 """ 13 14 import os 15 import re 16 import sys 17 import pkgutil 18 import logging 19 from cgi import escape 20 from threading import RLock 21 22 try: 23 import copyreg as copy_reg # python 3 24 except ImportError: 25 import copy_reg # python 2 26 27 from gluon.portalocker import read_locked, LockedFile 28 from utf8 import Utf8 29 30 from gluon.fileutils import listdir 31 from gluon.cfs import getcfs 32 from gluon.html import XML, xmlescape 33 from gluon.contrib.markmin.markmin2html import render, markmin_escape 34 from string import maketrans 35 36 __all__ = ['translator', 'findT', 'update_all_languages'] 37 38 ostat = os.stat 39 oslistdir = os.listdir 40 pjoin = os.path.join 41 pexists = os.path.exists 42 pdirname = os.path.dirname 43 isdir = os.path.isdir 44 45 DEFAULT_LANGUAGE = 'en' 46 DEFAULT_LANGUAGE_NAME = 'English' 47 48 # DEFAULT PLURAL-FORMS RULES: 49 # language doesn't use plural forms 50 DEFAULT_NPLURALS = 1 51 # only one singular/plural form is used 52 DEFAULT_GET_PLURAL_ID = lambda n: 0 53 # word is unchangeable 54 DEFAULT_CONSTRUCT_PLURAL_FORM = lambda word, plural_id: word 55 56 NUMBERS = (int, long, float) 57 58 # pattern to find T(blah blah blah) expressions 59 PY_STRING_LITERAL_RE = r'(?<=[^\w]T$)(?P<name>'\ 60 + r"[uU]?[rR]?(?:'''(?:[^']|'{1,2}(?!'))*''')|"\ 61 + r"(?:'(?:[^'\\]|\\.)*')|" + r'(?:"""(?:[^"]|"{1,2}(?!"))*""")|'\ 62 + r'(?:"(?:[^"\\]|\\.)*"))' 63 64 regex_translate = re.compile(PY_STRING_LITERAL_RE, re.DOTALL) 65 regex_param = re.compile(r'{(?P<s>.+?)}') 66 67 # pattern for a valid accept_language 68 regex_language = \ 69 re.compile('([a-z]{2,3}(?:\-[a-z]{2})?(?:\-[a-z]{2})?)(?:[,;]|$)') 70 regex_langfile = re.compile('^[a-z]{2,3}(-[a-z]{2})?\.py$') 71 regex_backslash = re.compile(r"\\([\\{}%])") 72 regex_plural = re.compile('%({.+?})') 73 regex_plural_dict = re.compile('^{(?P<w>[^()[\]][^()[\]]*?)\((?P<n>[^()\[\]]+)$}$') # %%{word(varname or number)} 74 regex_plural_tuple = re.compile( 75 '^{(?P<w>[^[\]()]+)(?:\[(?P<i>\d+)\])?}$') # %%{word[index]} or %%{word} 76 regex_plural_file = re.compile('^plural-[a-zA-Z]{2}(-[a-zA-Z]{2})?\.py$') 77

78 -def is_writable():

79 """ returns True if and only if the filesystem is writable """ 80 from gluon.settings import global_settings 81 return not global_settings.web2py_runtime_gae

82

83 -def safe_eval(text):

84 if text.strip(): 85 try: 86 import ast 87 return ast.literal_eval(text) 88 except ImportError: 89 return eval(text, {}, {}) 90 return None

91 92 # used as default filter in translator.M() 93 94

95 -def markmin(s):

96 def markmin_aux(m): 97 return '{%s}' % markmin_escape(m.group('s'))

98 return render(regex_param.sub(markmin_aux, s), 99 sep='br', autolinks=None, id_prefix='') 100 101 # UTF8 helper functions 102 103

104 -def upper_fun(s):

105 return unicode(s, 'utf-8').upper().encode('utf-8')

106 107

108 -def title_fun(s):

109 return unicode(s, 'utf-8').title().encode('utf-8')

110 111

112 -def cap_fun(s):

113 return unicode(s, 'utf-8').capitalize().encode('utf-8')

114 ttab_in = maketrans("\\%{}", '\x1c\x1d\x1e\x1f') 115 ttab_out = maketrans('\x1c\x1d\x1e\x1f', "\\%{}") 116 117 # cache of translated messages: 118 # global_language_cache: 119 # { 'languages/xx.py': 120 # ( {"def-message": "xx-message", 121 # ... 122 # "def-message": "xx-message"}, lock_object ) 123 # 'languages/yy.py': ( {dict}, lock_object ) 124 # ... 125 # } 126 127 global_language_cache = {} 128 129

130 -def get_from_cache(cache, val, fun):

131 lang_dict, lock = cache 132 lock.acquire() 133 try: 134 result = lang_dict.get(val) 135 finally: 136 lock.release() 137 if result: 138 return result 139 lock.acquire() 140 try: 141 result = lang_dict.setdefault(val, fun()) 142 finally: 143 lock.release() 144 return result

145 146

147 -def clear_cache(filename):

148 cache = global_language_cache.setdefault( 149 filename, ({}, RLock())) 150 lang_dict, lock = cache 151 lock.acquire() 152 try: 153 lang_dict.clear() 154 finally: 155 lock.release()

156 157

158 -def read_dict_aux(filename):

159 lang_text = read_locked(filename).replace('\r\n', '\n') 160 clear_cache(filename) 161 try: 162 return safe_eval(lang_text) or {} 163 except Exception: 164 e = sys.exc_info()[1] 165 status = 'Syntax error in %s (%s)' % (filename, e) 166 logging.error(status) 167 return {'__corrupted__': status}

168 169

170 -def read_dict(filename):

171 """ Returns dictionary with translation messages 172 """ 173 return getcfs('lang:' + filename, filename, 174 lambda: read_dict_aux(filename))

175 176

177 -def read_possible_plural_rules():

178 """ 179 Creates list of all possible plural rules files 180 The result is cached in PLURAL_RULES dictionary to increase speed 181 """ 182 plurals = {} 183 try: 184 import gluon.contrib.plural_rules as package 185 for importer, modname, ispkg in pkgutil.iter_modules(package.__path__): 186 if len(modname) == 2: 187 module = __import__(package.__name__ + '.' + modname, 188 fromlist=[modname]) 189 lang = modname 190 pname = modname + '.py' 191 nplurals = getattr(module, 'nplurals', DEFAULT_NPLURALS) 192 get_plural_id = getattr( 193 module, 'get_plural_id', 194 DEFAULT_GET_PLURAL_ID) 195 construct_plural_form = getattr( 196 module, 'construct_plural_form', 197 DEFAULT_CONSTRUCT_PLURAL_FORM) 198 plurals[lang] = (lang, nplurals, get_plural_id, 199 construct_plural_form) 200 except ImportError: 201 e = sys.exc_info()[1] 202 logging.warn('Unable to import plural rules: %s' % e) 203 return plurals

204 205 PLURAL_RULES = read_possible_plural_rules() 206 207

208 -def read_possible_languages_aux(langdir):

209 def get_lang_struct(lang, langcode, langname, langfile_mtime): 210 if lang == 'default': 211 real_lang = langcode.lower() 212 else: 213 real_lang = lang 214 (prules_langcode, 215 nplurals, 216 get_plural_id, 217 construct_plural_form 218 ) = PLURAL_RULES.get(real_lang[:2], ('default', 219 DEFAULT_NPLURALS, 220 DEFAULT_GET_PLURAL_ID, 221 DEFAULT_CONSTRUCT_PLURAL_FORM)) 222 if prules_langcode != 'default': 223 (pluraldict_fname, 224 pluraldict_mtime) = plurals.get(real_lang, 225 plurals.get(real_lang[:2], 226 ('plural-%s.py' % real_lang, 0))) 227 else: 228 pluraldict_fname = None 229 pluraldict_mtime = 0 230 return (langcode, # language code from !langcode! 231 langname, 232 # language name in national spelling from !langname! 233 langfile_mtime, # m_time of language file 234 pluraldict_fname, # name of plural dictionary file or None (when default.py is not exist) 235 pluraldict_mtime, # m_time of plural dictionary file or 0 if file is not exist 236 prules_langcode, # code of plural rules language or 'default' 237 nplurals, # nplurals for current language 238 get_plural_id, # get_plural_id() for current language 239 construct_plural_form) # construct_plural_form() for current language

240 241 plurals = {} 242 flist = oslistdir(langdir) if isdir(langdir) else [] 243 244 # scan languages directory for plural dict files: 245 for pname in flist: 246 if regex_plural_file.match(pname): 247 plurals[pname[7:-3]] = (pname, 248 ostat(pjoin(langdir, pname)).st_mtime) 249 langs = {} 250 # scan languages directory for langfiles: 251 for fname in flist: 252 if regex_langfile.match(fname) or fname == 'default.py': 253 fname_with_path = pjoin(langdir, fname) 254 d = read_dict(fname_with_path) 255 lang = fname[:-3] 256 langcode = d.get('!langcode!', lang if lang != 'default' 257 else DEFAULT_LANGUAGE) 258 langname = d.get('!langname!', langcode) 259 langfile_mtime = ostat(fname_with_path).st_mtime 260 langs[lang] = get_lang_struct(lang, langcode, 261 langname, langfile_mtime) 262 if 'default' not in langs: 263 # if default.py is not found, 264 # add DEFAULT_LANGUAGE as default language: 265 langs['default'] = get_lang_struct('default', DEFAULT_LANGUAGE, 266 DEFAULT_LANGUAGE_NAME, 0) 267 deflang = langs['default'] 268 deflangcode = deflang[0] 269 if deflangcode not in langs: 270 # create language from default.py: 271 langs[deflangcode] = deflang[:2] + (0,) + deflang[3:] 272 273 return langs 274 275

276 -def read_possible_languages(langpath):

277 return getcfs('langs:' + langpath, langpath, 278 lambda: read_possible_languages_aux(langpath))

279 280

281 -def read_plural_dict_aux(filename):

282 lang_text = read_locked(filename).replace('\r\n', '\n') 283 try: 284 return eval(lang_text) or {} 285 except Exception: 286 e = sys.exc_info()[1] 287 status = 'Syntax error in %s (%s)' % (filename, e) 288 logging.error(status) 289 return {'__corrupted__': status}

290 291

292 -def read_plural_dict(filename):

293 return getcfs('plurals:' + filename, filename, 294 lambda: read_plural_dict_aux(filename))

295 296

297 -def write_plural_dict(filename, contents):

298 if '__corrupted__' in contents: 299 return 300 fp = None 301 try: 302 fp = LockedFile(filename, 'w') 303 fp.write('#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n{\n# "singular form (0)": ["first plural form (1)", "second plural form (2)", ...],\n') 304 for key in sorted(contents, sort_function): 305 forms = '[' + ','.join([repr(Utf8(form)) 306 for form in contents[key]]) + ']' 307 fp.write('%s: %s,\n' % (repr(Utf8(key)), forms)) 308 fp.write('}\n') 309 except (IOError, OSError): 310 if is_writable(): 311 logging.warning('Unable to write to file %s' % filename) 312 return 313 finally: 314 if fp: 315 fp.close()

316

317 -def sort_function(x,y):

318 return cmp(unicode(x, 'utf-8').lower(), unicode(y, 'utf-8').lower())

319

320 -def write_dict(filename, contents):

321 if '__corrupted__' in contents: 322 return 323 fp = None 324 try: 325 fp = LockedFile(filename, 'w') 326 fp.write('# -*- coding: utf-8 -*-\n{\n') 327 for key in sorted(contents, sort_function): 328 fp.write('%s: %s,\n' % (repr(Utf8(key)), 329 repr(Utf8(contents[key])))) 330 fp.write('}\n') 331 except (IOError, OSError): 332 if is_writable(): 333 logging.warning('Unable to write to file %s' % filename) 334 return 335 finally: 336 if fp: 337 fp.close()

338 339

340 -class lazyT(object):

341 """ 342 Never to be called explicitly, returned by 343 translator.__call__() or translator.M() 344 """ 345 m = s = T = f = t = None 346 M = is_copy = False 347

348 - def __init__( 349 self, 350 message, 351 symbols={}, 352 T=None, 353 filter=None, 354 ftag=None, 355 M=False 356 ):

357 if isinstance(message, lazyT): 358 self.m = message.m 359 self.s = message.s 360 self.T = message.T 361 self.f = message.f 362 self.t = message.t 363 self.M = message.M 364 self.is_copy = True 365 else: 366 self.m = message 367 self.s = symbols 368 self.T = T 369 self.f = filter 370 self.t = ftag 371 self.M = M 372 self.is_copy = False

373

374 - def __repr__(self):

375 return "<lazyT %s>" % (repr(Utf8(self.m)), )

376

377 - def __str__(self):

378 return str(self.T.apply_filter(self.m, self.s, self.f, self.t) if self.M else 379 self.T.translate(self.m, self.s))

380

381 - def __eq__(self, other):

382 return str(self) == str(other)

383

384 - def __ne__(self, other):

385 return str(self) != str(other)

386

387 - def __add__(self, other):

388 return '%s%s' % (self, other)

389

390 - def __radd__(self, other):

391 return '%s%s' % (other, self)

392

393 - def __mul__(self, other):

394 return str(self) * other

395

396 - def __cmp__(self, other):

397 return cmp(str(self), str(other))

398

399 - def __hash__(self):

400 return hash(str(self))

401

402 - def __getattr__(self, name):

403 return getattr(str(self), name)

404

405 - def __getitem__(self, i):

406 return str(self)[i]

407

408 - def __getslice__(self, i, j):

409 return str(self)[i:j]

410

411 - def __iter__(self):

412 for c in str(self): 413 yield c

414

415 - def __len__(self):

416 return len(str(self))

417

418 - def xml(self):

419 return str(self) if self.M else escape(str(self))

420

421 - def encode(self, *a, **b):

422 return str(self).encode(*a, **b)

423

424 - def decode(self, *a, **b):

425 return str(self).decode(*a, **b)

426

427 - def read(self):

428 return str(self)

429

430 - def __mod__(self, symbols):

431 if self.is_copy: 432 return lazyT(self) 433 return lazyT(self.m, symbols, self.T, self.f, self.t, self.M)

434

435 -def pickle_lazyT(c):

436 return str, (c.xml(),)

437 438 copy_reg.pickle(lazyT, pickle_lazyT) 439

440 -class translator(object):

441 """ 442 This class is instantiated by gluon.compileapp.build_environment 443 as the T object 444 445 Example: 446 447 T.force(None) # turns off translation 448 T.force('fr, it') # forces web2py to translate using fr.py or it.py 449 450 T("Hello World") # translates "Hello World" using the selected file 451 452 Note: 453 - there is no need to force since, by default, T uses 454 http_accept_language to determine a translation file. 455 - en and en-en are considered different languages! 456 - if language xx-yy is not found force() probes other similar languages 457 using such algorithm: `xx-yy.py -> xx.py -> xx-yy*.py -> xx*.py` 458 """ 459

460 - def __init__(self, langpath, http_accept_language):

461 self.langpath = langpath 462 self.http_accept_language = http_accept_language 463 # filled in self.force(): 464 #------------------------ 465 # self.cache 466 # self.accepted_language 467 # self.language_file 468 # self.plural_language 469 # self.nplurals 470 # self.get_plural_id 471 # self.construct_plural_form 472 # self.plural_file 473 # self.plural_dict 474 # self.requested_languages 475 #---------------------------------------- 476 # filled in self.set_current_languages(): 477 #---------------------------------------- 478 # self.default_language_file 479 # self.default_t 480 # self.current_languages 481 self.set_current_languages() 482 self.lazy = True 483 self.otherTs = {} 484 self.filter = markmin 485 self.ftag = 'markmin' 486 self.ns = None

487

488 - def get_possible_languages_info(self, lang=None):

489 """ 490 Returns info for selected language or dictionary with all 491 possible languages info from `APP/languages/*.py` 492 It Returns: 493 494 - a tuple containing:: 495 496 langcode, langname, langfile_mtime, 497 pluraldict_fname, pluraldict_mtime, 498 prules_langcode, nplurals, 499 get_plural_id, construct_plural_form 500 501 or None 502 503 - if *lang* is NOT defined a dictionary with all possible 504 languages:: 505 506 { langcode(from filename): 507 ( langcode, # language code from !langcode! 508 langname, 509 # language name in national spelling from !langname! 510 langfile_mtime, # m_time of language file 511 pluraldict_fname,# name of plural dictionary file or None (when default.py is not exist) 512 pluraldict_mtime,# m_time of plural dictionary file or 0 if file is not exist 513 prules_langcode, # code of plural rules language or 'default' 514 nplurals, # nplurals for current language 515 get_plural_id, # get_plural_id() for current language 516 construct_plural_form) # construct_plural_form() for current language 517 } 518 519 Args: 520 lang (str): language 521 522 """ 523 info = read_possible_languages(self.langpath) 524 if lang: 525 info = info.get(lang) 526 return info

527

528 - def get_possible_languages(self):

529 """ Gets list of all possible languages for current application """ 530 return list(set(self.current_languages + 531 [lang for lang in read_possible_languages(self.langpath).iterkeys() 532 if lang != 'default']))

533

534 - def set_current_languages(self, *languages):

535 """ 536 Sets current AKA "default" languages 537 Setting one of this languages makes the force() function to turn 538 translation off 539 """ 540 if len(languages) == 1 and isinstance(languages[0], (tuple, list)): 541 languages = languages[0] 542 if not languages or languages[0] is None: 543 # set default language from default.py/DEFAULT_LANGUAGE 544 pl_info = self.get_possible_languages_info('default') 545 if pl_info[2] == 0: # langfile_mtime 546 # if languages/default.py is not found 547 self.default_language_file = self.langpath 548 self.default_t = {} 549 self.current_languages = [DEFAULT_LANGUAGE] 550 else: 551 self.default_language_file = pjoin(self.langpath, 552 'default.py') 553 self.default_t = read_dict(self.default_language_file) 554 self.current_languages = [pl_info[0]] # !langcode! 555 else: 556 self.current_languages = list(languages) 557 self.force(self.http_accept_language)

558

559 - def plural(self, word, n):

560 """ Gets plural form of word for number *n* 561 invoked from T()/T.M() in `%%{}` tag 562 563 Args: 564 word (str): word in singular 565 n (numeric): number plural form created for 566 567 Returns: 568 word (str): word in appropriate singular/plural form 569 570 Note: 571 "word" MUST be defined in current language (T.accepted_language) 572 573 """ 574 if int(n) == 1: 575 return word 576 elif word: 577 id = self.get_plural_id(abs(int(n))) 578 # id = 0 singular form 579 # id = 1 first plural form 580 # id = 2 second plural form 581 # etc. 582 if id != 0: 583 forms = self.plural_dict.get(word, []) 584 if len(forms) >= id: 585 # have this plural form: 586 return forms[id - 1] 587 else: 588 # guessing this plural form 589 forms += [''] * (self.nplurals - len(forms) - 1) 590 form = self.construct_plural_form(word, id) 591 forms[id - 1] = form 592 self.plural_dict[word] = forms 593 if is_writable() and self.plural_file: 594 write_plural_dict(self.plural_file, 595 self.plural_dict) 596 return form 597 return word

598

599 - def force(self, *languages):

600 """ 601 Selects language(s) for translation 602 603 if a list of languages is passed as a parameter, 604 the first language from this list that matches the ones 605 from the possible_languages dictionary will be 606 selected 607 608 default language will be selected if none 609 of them matches possible_languages. 610 """ 611 pl_info = read_possible_languages(self.langpath) 612 613 def set_plural(language): 614 """ 615 initialize plural forms subsystem 616 """ 617 lang_info = pl_info.get(language) 618 if lang_info: 619 (pname, 620 pmtime, 621 self.plural_language, 622 self.nplurals, 623 self.get_plural_id, 624 self.construct_plural_form 625 ) = lang_info[3:] 626 pdict = {} 627 if pname: 628 pname = pjoin(self.langpath, pname) 629 if pmtime != 0: 630 pdict = read_plural_dict(pname) 631 self.plural_file = pname 632 self.plural_dict = pdict 633 else: 634 self.plural_language = 'default' 635 self.nplurals = DEFAULT_NPLURALS 636 self.get_plural_id = DEFAULT_GET_PLURAL_ID 637 self.construct_plural_form = DEFAULT_CONSTRUCT_PLURAL_FORM 638 self.plural_file = None 639 self.plural_dict = {}

640 language = '' 641 if len(languages) == 1 and isinstance(languages[0], str): 642 languages = regex_language.findall(languages[0].lower()) 643 elif not languages or languages[0] is None: 644 languages = [] 645 self.requested_languages = languages = tuple(languages) 646 if languages: 647 all_languages = set(lang for lang in pl_info.iterkeys() 648 if lang != 'default') \ 649 | set(self.current_languages) 650 for lang in languages: 651 # compare "aa-bb" | "aa" from *language* parameter 652 # with strings from langlist using such alghorythm: 653 # xx-yy.py -> xx.py -> xx*.py 654 lang5 = lang[:5] 655 if lang5 in all_languages: 656 language = lang5 657 else: 658 lang2 = lang[:2] 659 if len(lang5) > 2 and lang2 in all_languages: 660 language = lang2 661 else: 662 for l in all_languages: 663 if l[:2] == lang2: 664 language = l 665 if language: 666 if language in self.current_languages: 667 break 668 self.language_file = pjoin(self.langpath, language + '.py') 669 self.t = read_dict(self.language_file) 670 self.cache = global_language_cache.setdefault( 671 self.language_file, 672 ({}, RLock())) 673 set_plural(language) 674 self.accepted_language = language 675 return languages 676 self.accepted_language = language 677 if not language: 678 if self.current_languages: 679 self.accepted_language = self.current_languages[0] 680 else: 681 self.accepted_language = DEFAULT_LANGUAGE 682 self.language_file = self.default_language_file 683 self.cache = global_language_cache.setdefault(self.language_file, 684 ({}, RLock())) 685 self.t = self.default_t 686 set_plural(self.accepted_language) 687 return languages

688

689 - def __call__(self, message, symbols={}, language=None, lazy=None, ns=None):

690 """ 691 get cached translated plain text message with inserted parameters(symbols) 692 if lazy==True lazyT object is returned 693 """ 694 if lazy is None: 695 lazy = self.lazy 696 if not language and not ns: 697 if lazy: 698 return lazyT(message, symbols, self) 699 else: 700 return self.translate(message, symbols) 701 else: 702 if ns: 703 if ns != self.ns: 704 self.langpath = os.path.join(self.langpath, ns) 705 if self.ns is None: 706 self.ns = ns 707 otherT = self.__get_otherT__(language, ns) 708 return otherT(message, symbols, lazy=lazy)

709

710 - def __get_otherT__(self, language=None, namespace=None):

711 if not language and not namespace: 712 raise Exception('Incorrect parameters') 713 714 if namespace: 715 if language: 716 index = '%s/%s' % (namespace, language) 717 else: 718 index = namespace 719 else: 720 index = language 721 try: 722 otherT = self.otherTs[index] 723 except KeyError: 724 otherT = self.otherTs[index] = translator(self.langpath, \ 725 self.http_accept_language) 726 if language: 727 otherT.force(language) 728 return otherT

729

730 - def apply_filter(self, message, symbols={}, filter=None, ftag=None):

731 def get_tr(message, prefix, filter): 732 s = self.get_t(message, prefix) 733 return filter(s) if filter else self.filter(s)

734 if filter: 735 prefix = '@' + (ftag or 'userdef') + '\x01' 736 else: 737 prefix = '@' + self.ftag + '\x01' 738 message = get_from_cache( 739 self.cache, prefix + message, 740 lambda: get_tr(message, prefix, filter)) 741 if symbols or symbols == 0 or symbols == "": 742 if isinstance(symbols, dict): 743 symbols.update( 744 (key, xmlescape(value).translate(ttab_in)) 745 for key, value in symbols.iteritems() 746 if not isinstance(value, NUMBERS)) 747 else: 748 if not isinstance(symbols, tuple): 749 symbols = (symbols,) 750 symbols = tuple( 751 value if isinstance(value, NUMBERS) 752 else xmlescape(value).translate(ttab_in) 753 for value in symbols) 754 message = self.params_substitution(message, symbols) 755 return XML(message.translate(ttab_out)) 756

757 - def M(self, message, symbols={}, language=None, 758 lazy=None, filter=None, ftag=None, ns=None):

759 """ 760 Gets cached translated markmin-message with inserted parametes 761 if lazy==True lazyT object is returned 762 """ 763 if lazy is None: 764 lazy = self.lazy 765 if not language and not ns: 766 if lazy: 767 return lazyT(message, symbols, self, filter, ftag, True) 768 else: 769 return self.apply_filter(message, symbols, filter, ftag) 770 else: 771 if ns: 772 self.langpath = os.path.join(self.langpath, ns) 773 otherT = self.__get_otherT__(language, ns) 774 return otherT.M(message, symbols, lazy=lazy)

775

776 - def get_t(self, message, prefix=''):

777 """ 778 Use ## to add a comment into a translation string 779 the comment can be useful do discriminate different possible 780 translations for the same string (for example different locations):: 781 782 T(' hello world ') -> ' hello world ' 783 T(' hello world ## token') -> ' hello world ' 784 T('hello ## world## token') -> 'hello ## world' 785 786 the ## notation is ignored in multiline strings and strings that 787 start with ##. This is needed to allow markmin syntax to be translated 788 """ 789 if isinstance(message, unicode): 790 message = message.encode('utf8') 791 if isinstance(prefix, unicode): 792 prefix = prefix.encode('utf8') 793 key = prefix + message 794 mt = self.t.get(key, None) 795 if mt is not None: 796 return mt 797 # we did not find a translation 798 if message.find('##') > 0 and not '\n' in message: 799 # remove comments 800 message = message.rsplit('##', 1)[0] 801 # guess translation same as original 802 self.t[key] = mt = self.default_t.get(key, message) 803 # update language file for latter translation 804 if is_writable() and \ 805 self.language_file != self.default_language_file: 806 write_dict(self.language_file, self.t) 807 return regex_backslash.sub( 808 lambda m: m.group(1).translate(ttab_in), mt)

809

810 - def params_substitution(self, message, symbols):

811 """ 812 Substitutes parameters from symbols into message using %. 813 also parse `%%{}` placeholders for plural-forms processing. 814 815 Returns: 816 string with parameters 817 818 Note: 819 *symbols* MUST BE OR tuple OR dict of parameters! 820 """ 821 def sub_plural(m): 822 """String in `%{}` is transformed by this rules: 823 If string starts with `\\`, `!` or `?` such transformations 824 take place:: 825 826 "!string of words" -> "String of word" (Capitalize) 827 "!!string of words" -> "String Of Word" (Title) 828 "!!!string of words" -> "STRING OF WORD" (Upper) 829 "\\!string of words" -> "!string of word" 830 (remove \\ and disable transformations) 831 "?word?number" -> "word" (return word, if number == 1) 832 "?number" or "??number" -> "" (remove number, 833 if number == 1) 834 "?word?number" -> "number" (if number != 1) 835 836 """ 837 def sub_tuple(m): 838 """ word[number], !word[number], !!word[number], !!!word[number] 839 word, !word, !!word, !!!word, ?word?number, ??number, ?number 840 ?word?word[number], ?word?[number], ??word[number] 841 """ 842 w, i = m.group('w', 'i') 843 c = w[0] 844 if c not in '!?': 845 return self.plural(w, symbols[int(i or 0)]) 846 elif c == '?': 847 (p1, sep, p2) = w[1:].partition("?") 848 part1 = p1 if sep else "" 849 (part2, sep, part3) = (p2 if sep else p1).partition("?") 850 if not sep: 851 part3 = part2 852 if i is None: 853 # ?[word]?number[?number] or ?number 854 if not part2: 855 return m.group(0) 856 num = int(part2) 857 else: 858 # ?[word]?word2[?word3][number] 859 num = int(symbols[int(i or 0)]) 860 return part1 if num == 1 else part3 if num == 0 else part2 861 elif w.startswith('!!!'): 862 word = w[3:] 863 fun = upper_fun 864 elif w.startswith('!!'): 865 word = w[2:] 866 fun = title_fun 867 else: 868 word = w[1:] 869 fun = cap_fun 870 if i is not None: 871 return fun(self.plural(word, symbols[int(i)])) 872 return fun(word)

873 874 def sub_dict(m): 875 """ word(var), !word(var), !!word(var), !!!word(var) 876 word(num), !word(num), !!word(num), !!!word(num) 877 ?word2(var), ?word1?word2(var), ?word1?word2?word0(var) 878 ?word2(num), ?word1?word2(num), ?word1?word2?word0(num) 879 """ 880 w, n = m.group('w', 'n') 881 c = w[0] 882 n = int(n) if n.isdigit() else symbols[n] 883 if c not in '!?': 884 return self.plural(w, n) 885 elif c == '?': 886 # ?[word1]?word2[?word0](var or num), ?[word1]?word2(var or num) or ?word2(var or num) 887 (p1, sep, p2) = w[1:].partition("?") 888 part1 = p1 if sep else "" 889 (part2, sep, part3) = (p2 if sep else p1).partition("?") 890 if not sep: 891 part3 = part2 892 num = int(n) 893 return part1 if num == 1 else part3 if num == 0 else part2 894 elif w.startswith('!!!'): 895 word = w[3:] 896 fun = upper_fun 897 elif w.startswith('!!'): 898 word = w[2:] 899 fun = title_fun 900 else: 901 word = w[1:] 902 fun = cap_fun 903 return fun(self.plural(word, n)) 904 905 s = m.group(1) 906 part = regex_plural_tuple.sub(sub_tuple, s) 907 if part == s: 908 part = regex_plural_dict.sub(sub_dict, s) 909 if part == s: 910 return m.group(0) 911 return part 912 message = message % symbols 913 message = regex_plural.sub(sub_plural, message) 914 return message 915

916 - def translate(self, message, symbols):

917 """ 918 Gets cached translated message with inserted parameters(symbols) 919 """ 920 message = get_from_cache(self.cache, message, 921 lambda: self.get_t(message)) 922 if symbols or symbols == 0 or symbols == "": 923 if isinstance(symbols, dict): 924 symbols.update( 925 (key, str(value).translate(ttab_in)) 926 for key, value in symbols.iteritems() 927 if not isinstance(value, NUMBERS)) 928 else: 929 if not isinstance(symbols, tuple): 930 symbols = (symbols,) 931 symbols = tuple( 932 value if isinstance(value, NUMBERS) 933 else str(value).translate(ttab_in) 934 for value in symbols) 935 message = self.params_substitution(message, symbols) 936 return message.translate(ttab_out)

937 938

939 -def findT(path, language=DEFAULT_LANGUAGE):

940 """ 941 Note: 942 Must be run by the admin app 943 """ 944 lang_file = pjoin(path, 'languages', language + '.py') 945 sentences = read_dict(lang_file) 946 mp = pjoin(path, 'models') 947 cp = pjoin(path, 'controllers') 948 vp = pjoin(path, 'views') 949 mop = pjoin(path, 'modules') 950 for filename in \ 951 listdir(mp, '^.+\.py$', 0) + listdir(cp, '^.+\.py$', 0)\ 952 + listdir(vp, '^.+\.html$', 0) + listdir(mop, '^.+\.py$', 0): 953 data = read_locked(filename) 954 items = regex_translate.findall(data) 955 for item in items: 956 try: 957 message = safe_eval(item) 958 except: 959 continue # silently ignore inproperly formatted strings 960 if not message.startswith('#') and not '\n' in message: 961 tokens = message.rsplit('##', 1) 962 else: 963 # this allows markmin syntax in translations 964 tokens = [message] 965 if len(tokens) == 2: 966 message = tokens[0].strip() + '##' + tokens[1].strip() 967 if message and not message in sentences: 968 sentences[message] = message 969 if not '!langcode!' in sentences: 970 sentences['!langcode!'] = ( 971 DEFAULT_LANGUAGE if language in ('default', DEFAULT_LANGUAGE) else language) 972 if not '!langname!' in sentences: 973 sentences['!langname!'] = ( 974 DEFAULT_LANGUAGE_NAME if language in ('default', DEFAULT_LANGUAGE) 975 else sentences['!langcode!']) 976 write_dict(lang_file, sentences)

977

978 -def update_all_languages(application_path):

979 """ 980 Note: 981 Must be run by the admin app 982 """ 983 path = pjoin(application_path, 'languages/') 984 for language in oslistdir(path): 985 if regex_langfile.match(language): 986 findT(application_path, language[:-3])

987 988 989 if __name__ == '__main__': 990 import doctest 991 doctest.testmod() 992

Source Code for Module gluon.languages