gettext.py 18.5 KB
Newer Older
1 2 3 4 5 6 7 8
"""Internationalization and localization support.

This module provides internationalization (I18N) and localization (L10N)
support for your Python programs by providing an interface to the GNU gettext
message catalog library.

I18N refers to the operation by which a program is made aware of multiple
languages.  L10N refers to the adaptation of your program, once
9
internationalized, to the local language and cultural habits.
10 11 12

"""

13 14
# This module represents the integration of work, contributions, feedback, and
# suggestions from the following people:
15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
#
# Martin von Loewis, who wrote the initial implementation of the underlying
# C-based libintlmodule (later renamed _gettext), along with a skeletal
# gettext.py implementation.
#
# Peter Funk, who wrote fintl.py, a fairly complete wrapper around intlmodule,
# which also included a pure-Python implementation to read .mo files if
# intlmodule wasn't available.
#
# James Henstridge, who also wrote a gettext.py module, which has some
# interesting, but currently unsupported experimental features: the notion of
# a Catalog class and instances, and the ability to add to a catalog file via
# a Python API.
#
# Barry Warsaw integrated these modules, wrote the .install() API and code,
# and conformed all C and Python code to Python's coding standards.
31 32 33 34
#
# Francois Pinard and Marc-Andre Lemburg also contributed valuably to this
# module.
#
35
# J. David Ibanez implemented plural forms. Bruno Haible fixed some bugs.
36
#
37 38 39 40 41 42 43 44 45 46
# TODO:
# - Lazy loading of .mo files.  Currently the entire catalog is loaded into
#   memory, but that's probably bad for large translated programs.  Instead,
#   the lexical sort of original strings in GNU .mo files should be exploited
#   to do binary searches and lazy initializations.  Or you might want to use
#   the undocumented double-hash algorithm for .mo files with hash tables, but
#   you'll need to study the GNU gettext code to do this.
#
# - Support Solaris .mo file formats.  Unfortunately, we've been unable to
#   find this format documented anywhere.
47

48

49
import locale, copy, os, re, struct, sys
50
from errno import ENOENT
51

52

53 54 55 56
__all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',
           'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
           'dgettext', 'dngettext', 'gettext', 'ngettext',
           ]
57

58
_default_localedir = os.path.join(sys.prefix, 'share', 'locale')
59 60


61 62 63 64 65 66 67 68 69 70 71 72 73 74 75
def test(condition, true, false):
    """
    Implements the C expression:

      condition ? true : false

    Required to correctly interpret plural forms.
    """
    if condition:
        return true
    else:
        return false


def c2py(plural):
Barry Warsaw's avatar
Barry Warsaw committed
76 77
    """Gets a C expression as used in PO files for plural forms and returns a
    Python lambda function that implements an equivalent expression.
78 79
    """
    # Security check, allow only the "n" identifier
80
    from io import StringIO
81 82
    import token, tokenize
    tokens = tokenize.generate_tokens(StringIO(plural).readline)
83
    try:
Barry Warsaw's avatar
Barry Warsaw committed
84
        danger = [x for x in tokens if x[0] == token.NAME and x[1] != 'n']
85
    except tokenize.TokenError:
86
        raise ValueError('plural forms expression error, maybe unbalanced parenthesis')
87 88
    else:
        if danger:
89
            raise ValueError('plural forms expression could be dangerous')
90 91 92 93 94

    # Replace some C operators by their Python equivalents
    plural = plural.replace('&&', ' and ')
    plural = plural.replace('||', ' or ')

95 96
    expr = re.compile(r'\!([^=])')
    plural = expr.sub(' not \\1', plural)
97 98 99 100 101 102 103 104 105 106 107 108 109 110

    # Regular expression and replacement function used to transform
    # "a?b:c" to "test(a,b,c)".
    expr = re.compile(r'(.*?)\?(.*?):(.*)')
    def repl(x):
        return "test(%s, %s, %s)" % (x.group(1), x.group(2),
                                     expr.sub(repl, x.group(3)))

    # Code to transform the plural expression, taking care of parentheses
    stack = ['']
    for c in plural:
        if c == '(':
            stack.append('')
        elif c == ')':
111 112 113 114
            if len(stack) == 1:
                # Actually, we never reach this code, because unbalanced
                # parentheses get caught in the security check at the
                # beginning.
115
                raise ValueError('unbalanced parenthesis in plural form')
116 117 118 119 120 121 122 123 124
            s = expr.sub(repl, stack.pop())
            stack[-1] += '(%s)' % s
        else:
            stack[-1] += c
    plural = expr.sub(repl, stack.pop())

    return eval('lambda n: int(%s)' % plural)


Tim Peters's avatar
Tim Peters committed
125

126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167
def _expand_lang(locale):
    from locale import normalize
    locale = normalize(locale)
    COMPONENT_CODESET   = 1 << 0
    COMPONENT_TERRITORY = 1 << 1
    COMPONENT_MODIFIER  = 1 << 2
    # split up the locale into its base components
    mask = 0
    pos = locale.find('@')
    if pos >= 0:
        modifier = locale[pos:]
        locale = locale[:pos]
        mask |= COMPONENT_MODIFIER
    else:
        modifier = ''
    pos = locale.find('.')
    if pos >= 0:
        codeset = locale[pos:]
        locale = locale[:pos]
        mask |= COMPONENT_CODESET
    else:
        codeset = ''
    pos = locale.find('_')
    if pos >= 0:
        territory = locale[pos:]
        locale = locale[:pos]
        mask |= COMPONENT_TERRITORY
    else:
        territory = ''
    language = locale
    ret = []
    for i in range(mask+1):
        if not (i & ~mask):  # if all components for this combo exist ...
            val = language
            if i & COMPONENT_TERRITORY: val += territory
            if i & COMPONENT_CODESET:   val += codeset
            if i & COMPONENT_MODIFIER:  val += modifier
            ret.append(val)
    ret.reverse()
    return ret


Tim Peters's avatar
Tim Peters committed
168

169 170 171
class NullTranslations:
    def __init__(self, fp=None):
        self._info = {}
172
        self._charset = None
173
        self._output_charset = None
174
        self._fallback = None
175
        if fp is not None:
176
            self._parse(fp)
177

178 179 180
    def _parse(self, fp):
        pass

181 182 183 184 185 186
    def add_fallback(self, fallback):
        if self._fallback:
            self._fallback.add_fallback(fallback)
        else:
            self._fallback = fallback

187
    def gettext(self, message):
188 189
        if self._fallback:
            return self._fallback.gettext(message)
190 191
        return message

192 193 194 195 196
    def lgettext(self, message):
        if self._fallback:
            return self._fallback.lgettext(message)
        return message

197 198 199 200 201 202 203 204
    def ngettext(self, msgid1, msgid2, n):
        if self._fallback:
            return self._fallback.ngettext(msgid1, msgid2, n)
        if n == 1:
            return msgid1
        else:
            return msgid2

205 206 207 208 209 210 211 212
    def lngettext(self, msgid1, msgid2, n):
        if self._fallback:
            return self._fallback.lngettext(msgid1, msgid2, n)
        if n == 1:
            return msgid1
        else:
            return msgid2

213
    def ugettext(self, message):
214 215
        if self._fallback:
            return self._fallback.ugettext(message)
216
        return str(message)
217

218 219 220 221
    def ungettext(self, msgid1, msgid2, n):
        if self._fallback:
            return self._fallback.ungettext(msgid1, msgid2, n)
        if n == 1:
222
            return str(msgid1)
223
        else:
224
            return str(msgid2)
225

226 227 228 229 230 231
    def info(self):
        return self._info

    def charset(self):
        return self._charset

232 233 234 235 236 237
    def output_charset(self):
        return self._output_charset

    def set_output_charset(self, charset):
        self._output_charset = charset

238
    def install(self, str=False, names=None):
239 240
        import builtins
        builtins.__dict__['_'] = str and self.ugettext or self.gettext
241 242
        if hasattr(names, "__contains__"):
            if "gettext" in names:
243
                builtins.__dict__['gettext'] = builtins.__dict__['_']
244
            if "ngettext" in names:
245
                builtins.__dict__['ngettext'] = (str and self.ungettext
246 247
                                                             or self.ngettext)
            if "lgettext" in names:
248
                builtins.__dict__['lgettext'] = self.lgettext
249
            if "lngettext" in names:
250
                builtins.__dict__['lngettext'] = self.lngettext
251 252 253 254


class GNUTranslations(NullTranslations):
    # Magic number of .mo files
255 256
    LE_MAGIC = 0x950412de
    BE_MAGIC = 0xde120495
257 258 259 260 261 262 263

    def _parse(self, fp):
        """Override this method to support alternative .mo formats."""
        unpack = struct.unpack
        filename = getattr(fp, 'name', '')
        # Parse the .mo file header, which consists of 5 little endian 32
        # bit words.
264
        self._catalog = catalog = {}
265
        self.plural = lambda n: int(n != 1) # germanic plural by default
266
        buf = fp.read()
267
        buflen = len(buf)
268
        # Are we big endian or little endian?
269
        magic = unpack('<I', buf[:4])[0]
270
        if magic == self.LE_MAGIC:
271 272
            version, msgcount, masteridx, transidx = unpack('<4I', buf[4:20])
            ii = '<II'
273
        elif magic == self.BE_MAGIC:
274 275
            version, msgcount, masteridx, transidx = unpack('>4I', buf[4:20])
            ii = '>II'
276
        else:
277 278 279
            raise IOError(0, 'Bad magic number', filename)
        # Now put all messages from the .mo file buffer into the catalog
        # dictionary.
280
        for i in range(0, msgcount):
281
            mlen, moff = unpack(ii, buf[masteridx:masteridx+8])
282
            mend = moff + mlen
283
            tlen, toff = unpack(ii, buf[transidx:transidx+8])
284
            tend = toff + tlen
285
            if mend < buflen and tend < buflen:
286
                msg = buf[moff:mend]
287
                tmsg = buf[toff:tend]
288 289
            else:
                raise IOError(0, 'File is corrupt', filename)
290
            # See if we're looking at GNU .mo conventions for metadata
291
            if mlen == 0:
292
                # Catalog description
293
                lastk = k = None
294
                for b_item in tmsg.split('\n'.encode("ascii")):
295
                    item = b_item.decode().strip()
296 297
                    if not item:
                        continue
298 299 300 301 302 303 304 305
                    if ':' in item:
                        k, v = item.split(':', 1)
                        k = k.strip().lower()
                        v = v.strip()
                        self._info[k] = v
                        lastk = k
                    elif lastk:
                        self._info[lastk] += '\n' + item
306 307
                    if k == 'content-type':
                        self._charset = v.split('charset=')[1]
308 309 310 311
                    elif k == 'plural-forms':
                        v = v.split(';')
                        plural = v[1].split('plural=')[1]
                        self.plural = c2py(plural)
Barry Warsaw's avatar
Barry Warsaw committed
312 313 314 315 316 317 318 319 320
            # Note: we unconditionally convert both msgids and msgstrs to
            # Unicode using the character encoding specified in the charset
            # parameter of the Content-Type header.  The gettext documentation
            # strongly encourages msgids to be us-ascii, but some appliations
            # require alternative encodings (e.g. Zope's ZCML and ZPT).  For
            # traditional gettext applications, the msgid conversion will
            # cause no problems since us-ascii should always be a subset of
            # the charset encoding.  We may want to fall back to 8-bit msgids
            # if the Unicode conversion fails.
321
            if b'\x00' in msg:
322
                # Plural forms
323 324
                msgid1, msgid2 = msg.split(b'\x00')
                tmsg = tmsg.split(b'\x00')
Barry Warsaw's avatar
Barry Warsaw committed
325
                if self._charset:
326 327
                    msgid1 = str(msgid1, self._charset)
                    tmsg = [str(x, self._charset) for x in tmsg]
328 329 330
                else:
                    msgid1 = str(msgid1)
                    tmsg = [str(x) for x in tmsg]
331 332 333
                for i in range(len(tmsg)):
                    catalog[(msgid1, i)] = tmsg[i]
            else:
Barry Warsaw's avatar
Barry Warsaw committed
334
                if self._charset:
335 336
                    msg = str(msg, self._charset)
                    tmsg = str(tmsg, self._charset)
337 338 339
                else:
                    msg = str(msg)
                    tmsg = str(tmsg)
340
                catalog[msg] = tmsg
341
            # advance to next entry in the seek tables
342 343
            masteridx += 8
            transidx += 8
344

345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
    def lgettext(self, message):
        missing = object()
        tmsg = self._catalog.get(message, missing)
        if tmsg is missing:
            if self._fallback:
                return self._fallback.lgettext(message)
            return message
        if self._output_charset:
            return tmsg.encode(self._output_charset)
        return tmsg.encode(locale.getpreferredencoding())

    def lngettext(self, msgid1, msgid2, n):
        try:
            tmsg = self._catalog[(msgid1, self.plural(n))]
            if self._output_charset:
                return tmsg.encode(self._output_charset)
            return tmsg.encode(locale.getpreferredencoding())
        except KeyError:
            if self._fallback:
                return self._fallback.lngettext(msgid1, msgid2, n)
            if n == 1:
                return msgid1
            else:
                return msgid2

370
    def ugettext(self, message):
371 372 373
        missing = object()
        tmsg = self._catalog.get(message, missing)
        if tmsg is missing:
374 375
            if self._fallback:
                return self._fallback.ugettext(message)
376
            return str(message)
377
        return tmsg
378

379 380
    gettext = ugettext

381 382 383 384 385 386 387
    def ungettext(self, msgid1, msgid2, n):
        try:
            tmsg = self._catalog[(msgid1, self.plural(n))]
        except KeyError:
            if self._fallback:
                return self._fallback.ungettext(msgid1, msgid2, n)
            if n == 1:
388
                tmsg = str(msgid1)
389
            else:
390
                tmsg = str(msgid2)
391
        return tmsg
392

393 394
    ngettext = ungettext

Tim Peters's avatar
Tim Peters committed
395

396
# Locate a .mo file using the gettext strategy
397
def find(domain, localedir=None, languages=None, all=0):
398 399
    # Get some reasonable defaults for arguments that were not supplied
    if localedir is None:
400
        localedir = _default_localedir
401 402 403 404 405 406 407 408 409
    if languages is None:
        languages = []
        for envar in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'):
            val = os.environ.get(envar)
            if val:
                languages = val.split(':')
                break
        if 'C' not in languages:
            languages.append('C')
410
    # now normalize and expand the languages
411
    nelangs = []
412 413
    for lang in languages:
        for nelang in _expand_lang(lang):
414 415
            if nelang not in nelangs:
                nelangs.append(nelang)
416
    # select a language
417 418 419 420
    if all:
        result = []
    else:
        result = None
421
    for lang in nelangs:
422 423
        if lang == 'C':
            break
424
        mofile = os.path.join(localedir, lang, 'LC_MESSAGES', '%s.mo' % domain)
425
        if os.path.exists(mofile):
426 427 428 429 430
            if all:
                result.append(mofile)
            else:
                return mofile
    return result
431 432


Tim Peters's avatar
Tim Peters committed
433

434 435 436
# a mapping between absolute .mo file path and Translation object
_translations = {}

437
def translation(domain, localedir=None, languages=None,
438
                class_=None, fallback=False, codeset=None):
439 440
    if class_ is None:
        class_ = GNUTranslations
441
    mofiles = find(domain, localedir, languages, all=1)
Barry Warsaw's avatar
Barry Warsaw committed
442
    if not mofiles:
443 444
        if fallback:
            return NullTranslations()
445 446
        raise IOError(ENOENT, 'No translation file found for domain', domain)
    # TBD: do we need to worry about the file pointer getting collected?
447 448
    # Avoid opening, reading, and parsing the .mo file after it's been done
    # once.
449 450 451 452 453 454
    result = None
    for mofile in mofiles:
        key = os.path.abspath(mofile)
        t = _translations.get(key)
        if t is None:
            t = _translations.setdefault(key, class_(open(mofile, 'rb')))
455 456 457
        # Copy the translation object to allow setting fallbacks and
        # output charset. All other instance data is shared with the
        # cached object.
458
        t = copy.copy(t)
459 460
        if codeset:
            t.set_output_charset(codeset)
461 462 463 464 465
        if result is None:
            result = t
        else:
            result.add_fallback(t)
    return result
466

Tim Peters's avatar
Tim Peters committed
467

468
def install(domain, localedir=None, str=False, codeset=None, names=None):
469
    t = translation(domain, localedir, fallback=True, codeset=codeset)
470
    t.install(str, names)
471 472


Tim Peters's avatar
Tim Peters committed
473

474 475
# a mapping b/w domains and locale directories
_localedirs = {}
476 477
# a mapping b/w domains and codesets
_localecodesets = {}
478 479
# current global domain, `messages' used for compatibility w/ GNU gettext
_current_domain = 'messages'
480 481 482 483


def textdomain(domain=None):
    global _current_domain
484
    if domain is not None:
485
        _current_domain = domain
486
    return _current_domain
487 488


489 490 491 492 493
def bindtextdomain(domain, localedir=None):
    global _localedirs
    if localedir is not None:
        _localedirs[domain] = localedir
    return _localedirs.get(domain, _default_localedir)
494 495


496 497 498 499 500 501 502
def bind_textdomain_codeset(domain, codeset=None):
    global _localecodesets
    if codeset is not None:
        _localecodesets[domain] = codeset
    return _localecodesets.get(domain)


503
def dgettext(domain, message):
504
    try:
505 506
        t = translation(domain, _localedirs.get(domain, None),
                        codeset=_localecodesets.get(domain))
507 508 509
    except IOError:
        return message
    return t.gettext(message)
Tim Peters's avatar
Tim Peters committed
510

511 512 513 514 515 516 517
def ldgettext(domain, message):
    try:
        t = translation(domain, _localedirs.get(domain, None),
                        codeset=_localecodesets.get(domain))
    except IOError:
        return message
    return t.lgettext(message)
518

519 520
def dngettext(domain, msgid1, msgid2, n):
    try:
521 522
        t = translation(domain, _localedirs.get(domain, None),
                        codeset=_localecodesets.get(domain))
523 524 525 526 527 528 529
    except IOError:
        if n == 1:
            return msgid1
        else:
            return msgid2
    return t.ngettext(msgid1, msgid2, n)

530 531 532 533 534 535 536 537 538 539
def ldngettext(domain, msgid1, msgid2, n):
    try:
        t = translation(domain, _localedirs.get(domain, None),
                        codeset=_localecodesets.get(domain))
    except IOError:
        if n == 1:
            return msgid1
        else:
            return msgid2
    return t.lngettext(msgid1, msgid2, n)
540

541 542
def gettext(message):
    return dgettext(_current_domain, message)
543

544 545
def lgettext(message):
    return ldgettext(_current_domain, message)
546

547 548 549
def ngettext(msgid1, msgid2, n):
    return dngettext(_current_domain, msgid1, msgid2, n)

550 551
def lngettext(msgid1, msgid2, n):
    return ldngettext(_current_domain, msgid1, msgid2, n)
552

553
# dcgettext() has been deemed unnecessary and is not implemented.
554

555 556 557 558 559 560 561
# James Henstridge's Catalog constructor from GNOME gettext.  Documented usage
# was:
#
#    import gettext
#    cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR)
#    _ = cat.gettext
#    print _('Hello World')
562

563 564 565
# The resulting catalog object currently don't support access through a
# dictionary API, which was supported (but apparently unused) in GNOME
# gettext.
566

567
Catalog = translation