base64.py 20 KB
Newer Older
1
#! /usr/bin/env python3
Guido van Rossum's avatar
Guido van Rossum committed
2

3
"""Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings"""
4

5 6
# Modified 04-Oct-1995 by Jack Jansen to use binascii module
# Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support
7
# Modified 22-May-2007 by Guido van Rossum to use bytes everywhere
8

9 10
import re
import struct
11
import binascii
12

13 14

__all__ = [
15
    # Legacy interface exports traditional RFC 2045 Base64 encodings
16
    'encode', 'decode', 'encodebytes', 'decodebytes',
17 18 19
    # Generalized interface for other encodings
    'b64encode', 'b64decode', 'b32encode', 'b32decode',
    'b16encode', 'b16decode',
20 21
    # Base85 and Ascii85 encodings
    'b85encode', 'b85decode', 'a85encode', 'a85decode',
22 23 24 25 26 27 28 29 30 31
    # Standard Base64 encoding
    'standard_b64encode', 'standard_b64decode',
    # Some common Base64 alternatives.  As referenced by RFC 3458, see thread
    # starting at:
    #
    # http://zgp.org/pipermail/p2p-hackers/2001-September/000316.html
    'urlsafe_b64encode', 'urlsafe_b64decode',
    ]


32
bytes_types = (bytes, bytearray)  # Types acceptable as binary data
33

34 35 36 37 38 39
def _bytes_from_decode_data(s):
    if isinstance(s, str):
        try:
            return s.encode('ascii')
        except UnicodeEncodeError:
            raise ValueError('string argument should contain only ASCII characters')
40
    if isinstance(s, bytes_types):
41
        return s
42 43 44 45 46
    try:
        return memoryview(s).tobytes()
    except TypeError:
        raise TypeError("argument should be a bytes-like object or ASCII "
                        "string, not %r" % s.__class__.__name__) from None
47

48

49 50 51
# Base64 encoding/decoding uses binascii

def b64encode(s, altchars=None):
52
    """Encode the bytes-like object s using Base64 and return a bytes object.
53

54 55 56
    Optional altchars should be a byte string of length 2 which specifies an
    alternative alphabet for the '+' and '/' characters.  This allows an
    application to e.g. generate url or filesystem safe Base64 strings.
57
    """
58
    encoded = binascii.b2a_base64(s, newline=False)
59
    if altchars is not None:
60
        assert len(altchars) == 2, repr(altchars)
61
        return encoded.translate(bytes.maketrans(b'+/', altchars))
62 63 64
    return encoded


65
def b64decode(s, altchars=None, validate=False):
66
    """Decode the Base64 encoded bytes-like object or ASCII string s.
67

68 69 70
    Optional altchars must be a bytes-like object or ASCII string of length 2
    which specifies the alternative alphabet used instead of the '+' and '/'
    characters.
71

72 73
    The result is returned as a bytes object.  A binascii.Error is raised if
    s is incorrectly padded.
74

75 76 77 78
    If validate is False (the default), characters that are neither in the
    normal base-64 alphabet nor the alternative alphabet are discarded prior
    to the padding check.  If validate is True, these non-alphabet characters
    in the input result in a binascii.Error.
79
    """
80
    s = _bytes_from_decode_data(s)
81
    if altchars is not None:
82
        altchars = _bytes_from_decode_data(altchars)
83
        assert len(altchars) == 2, repr(altchars)
84
        s = s.translate(bytes.maketrans(altchars, b'+/'))
85 86
    if validate and not re.match(b'^[A-Za-z0-9+/]*={0,2}$', s):
        raise binascii.Error('Non-base64 digit found')
87
    return binascii.a2b_base64(s)
88 89 90


def standard_b64encode(s):
91
    """Encode bytes-like object s using the standard Base64 alphabet.
92

93
    The result is returned as a bytes object.
94 95 96 97
    """
    return b64encode(s)

def standard_b64decode(s):
98
    """Decode bytes encoded with the standard Base64 alphabet.
99

100 101 102 103
    Argument s is a bytes-like object or ASCII string to decode.  The result
    is returned as a bytes object.  A binascii.Error is raised if the input
    is incorrectly padded.  Characters that are not in the standard alphabet
    are discarded prior to the padding check.
104 105 106
    """
    return b64decode(s)

107 108 109 110

_urlsafe_encode_translation = bytes.maketrans(b'+/', b'-_')
_urlsafe_decode_translation = bytes.maketrans(b'-_', b'+/')

111
def urlsafe_b64encode(s):
112
    """Encode bytes using the URL- and filesystem-safe Base64 alphabet.
113

114 115
    Argument s is a bytes-like object to encode.  The result is returned as a
    bytes object.  The alphabet uses '-' instead of '+' and '_' instead of
116
    '/'.
117
    """
118
    return b64encode(s).translate(_urlsafe_encode_translation)
119 120

def urlsafe_b64decode(s):
121
    """Decode bytes using the URL- and filesystem-safe Base64 alphabet.
122

123 124 125 126 127
    Argument s is a bytes-like object or ASCII string to decode.  The result
    is returned as a bytes object.  A binascii.Error is raised if the input
    is incorrectly padded.  Characters that are not in the URL-safe base-64
    alphabet, and are not a plus '+' or slash '/', are discarded prior to the
    padding check.
128 129 130

    The alphabet uses '-' instead of '+' and '_' instead of '/'.
    """
131 132 133
    s = _bytes_from_decode_data(s)
    s = s.translate(_urlsafe_decode_translation)
    return b64decode(s)
134 135


136

137
# Base32 encoding/decoding must be done in Python
138
_b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'
139 140
_b32tab2 = None
_b32rev = None
141 142

def b32encode(s):
143
    """Encode the bytes-like object s using Base32 and return a bytes object.
144
    """
145 146 147 148 149 150 151 152
    global _b32tab2
    # Delay the initialization of the table to not waste memory
    # if the function is never called
    if _b32tab2 is None:
        b32tab = [bytes((i,)) for i in _b32alphabet]
        _b32tab2 = [a + b for a in b32tab for b in b32tab]
        b32tab = None

153
    if not isinstance(s, bytes_types):
154
        s = memoryview(s).tobytes()
155
    leftover = len(s) % 5
156 157
    # Pad the last quantum with zero bits if necessary
    if leftover:
158
        s = s + b'\0' * (5 - leftover)  # Don't use += !
159
    encoded = bytearray()
160 161 162 163 164 165 166 167 168
    from_bytes = int.from_bytes
    b32tab2 = _b32tab2
    for i in range(0, len(s), 5):
        c = from_bytes(s[i: i + 5], 'big')
        encoded += (b32tab2[c >> 30] +           # bits 1 - 10
                    b32tab2[(c >> 20) & 0x3ff] + # bits 11 - 20
                    b32tab2[(c >> 10) & 0x3ff] + # bits 21 - 30
                    b32tab2[c & 0x3ff]           # bits 31 - 40
                   )
169 170
    # Adjust for any leftover partial quanta
    if leftover == 1:
171
        encoded[-6:] = b'======'
172
    elif leftover == 2:
173
        encoded[-4:] = b'===='
174
    elif leftover == 3:
175
        encoded[-3:] = b'==='
176
    elif leftover == 4:
177 178
        encoded[-1:] = b'='
    return bytes(encoded)
179 180

def b32decode(s, casefold=False, map01=None):
181
    """Decode the Base32 encoded bytes-like object or ASCII string s.
182

183 184
    Optional casefold is a flag specifying whether a lowercase alphabet is
    acceptable as input.  For security purposes, the default is False.
185 186 187 188 189 190 191 192 193

    RFC 3548 allows for optional mapping of the digit 0 (zero) to the
    letter O (oh), and for optional mapping of the digit 1 (one) to
    either the letter I (eye) or letter L (el).  The optional argument
    map01 when not None, specifies which letter the digit 1 should be
    mapped to (when map01 is not None, the digit 0 is always mapped to
    the letter O).  For security purposes the default is None, so that
    0 and 1 are not allowed in the input.

194
    The result is returned as a bytes object.  A binascii.Error is raised if
195 196
    the input is incorrectly padded or if there are non-alphabet
    characters present in the input.
197
    """
198 199 200 201 202
    global _b32rev
    # Delay the initialization of the table to not waste memory
    # if the function is never called
    if _b32rev is None:
        _b32rev = {v: k for k, v in enumerate(_b32alphabet)}
203
    s = _bytes_from_decode_data(s)
204
    if len(s) % 8:
205
        raise binascii.Error('Incorrect padding')
206 207 208
    # Handle section 2.4 zero and one mapping.  The flag map01 will be either
    # False, or the character to map the digit 1 (one) to.  It should be
    # either L (el) or I (eye).
209
    if map01 is not None:
210
        map01 = _bytes_from_decode_data(map01)
211
        assert len(map01) == 1, repr(map01)
212
        s = s.translate(bytes.maketrans(b'01', b'O' + map01))
213
    if casefold:
214
        s = s.upper()
215 216 217
    # Strip off pad characters from the right.  We need to count the pad
    # characters because this will tell us how many null bytes to remove from
    # the end of the decoded string.
218 219 220
    l = len(s)
    s = s.rstrip(b'=')
    padchars = l - len(s)
221
    # Now decode the full quanta
222 223 224 225 226 227 228 229 230
    decoded = bytearray()
    b32rev = _b32rev
    for i in range(0, len(s), 8):
        quanta = s[i: i + 8]
        acc = 0
        try:
            for c in quanta:
                acc = (acc << 5) + b32rev[c]
        except KeyError:
231
            raise binascii.Error('Non-base32 digit found') from None
232
        decoded += acc.to_bytes(5, 'big')
233
    # Process the last, partial quanta
234 235 236 237 238 239 240 241 242 243 244 245 246 247
    if padchars:
        acc <<= 5 * padchars
        last = acc.to_bytes(5, 'big')
        if padchars == 1:
            decoded[-5:] = last[:-1]
        elif padchars == 3:
            decoded[-5:] = last[:-2]
        elif padchars == 4:
            decoded[-5:] = last[:-3]
        elif padchars == 6:
            decoded[-5:] = last[:-4]
        else:
            raise binascii.Error('Incorrect padding')
    return bytes(decoded)
248 249


250

251 252 253 254
# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns
# lowercase.  The RFC also recommends against accepting input case
# insensitively.
def b16encode(s):
255
    """Encode the bytes-like object s using Base16 and return a bytes object.
256
    """
257
    return binascii.hexlify(s).upper()
258 259 260


def b16decode(s, casefold=False):
261
    """Decode the Base16 encoded bytes-like object or ASCII string s.
262

263 264
    Optional casefold is a flag specifying whether a lowercase alphabet is
    acceptable as input.  For security purposes, the default is False.
265

266 267 268
    The result is returned as a bytes object.  A binascii.Error is raised if
    s is incorrectly padded or if there are non-alphabet characters present
    in the input.
269
    """
270
    s = _bytes_from_decode_data(s)
271
    if casefold:
272
        s = s.upper()
273
    if re.search(b'[^0-9A-F]', s):
274
        raise binascii.Error('Non-base16 digit found')
275 276
    return binascii.unhexlify(s)

277 278 279
#
# Ascii85 encoding/decoding
#
280

281 282 283 284 285
_a85chars = None
_a85chars2 = None
_A85START = b"<~"
_A85END = b"~>"

286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310
def _85encode(b, chars, chars2, pad=False, foldnuls=False, foldspaces=False):
    # Helper function for a85encode and b85encode
    if not isinstance(b, bytes_types):
        b = memoryview(b).tobytes()

    padding = (-len(b)) % 4
    if padding:
        b = b + b'\0' * padding
    words = struct.Struct('!%dI' % (len(b) // 4)).unpack(b)

    chunks = [b'z' if foldnuls and not word else
              b'y' if foldspaces and word == 0x20202020 else
              (chars2[word // 614125] +
               chars2[word // 85 % 7225] +
               chars[word % 85])
              for word in words]

    if padding and not pad:
        if chunks[-1] == b'z':
            chunks[-1] = chars[0] * 5
        chunks[-1] = chunks[-1][:-padding]

    return b''.join(chunks)

def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False):
311
    """Encode bytes-like object b using Ascii85 and return a bytes object.
312 313 314 315 316

    foldspaces is an optional flag that uses the special short sequence 'y'
    instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This
    feature is not supported by the "standard" Adobe encoding.

317
    wrapcol controls whether the output should have newline (b'\\n') characters
318 319 320
    added to it. If this is non-zero, each output line will be at most this
    many characters long.

321
    pad controls whether the input is padded to a multiple of 4 before
322 323 324 325 326
    encoding. Note that the btoa implementation always pads.

    adobe controls whether the encoded byte sequence is framed with <~ and ~>,
    which is used by the Adobe implementation.
    """
327 328 329 330 331 332 333
    global _a85chars, _a85chars2
    # Delay the initialization of tables to not waste memory
    # if the function is never called
    if _a85chars is None:
        _a85chars = [bytes((i,)) for i in range(33, 118)]
        _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars]

334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
    result = _85encode(b, _a85chars, _a85chars2, pad, True, foldspaces)

    if adobe:
        result = _A85START + result
    if wrapcol:
        wrapcol = max(2 if adobe else 1, wrapcol)
        chunks = [result[i: i + wrapcol]
                  for i in range(0, len(result), wrapcol)]
        if adobe:
            if len(chunks[-1]) + 2 > wrapcol:
                chunks.append(b'')
        result = b'\n'.join(chunks)
    if adobe:
        result += _A85END

    return result

def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'):
352
    """Decode the Ascii85 encoded bytes-like object or ASCII string b.
353 354 355 356 357 358 359 360 361 362 363

    foldspaces is a flag that specifies whether the 'y' short sequence should be
    accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is
    not supported by the "standard" Adobe encoding.

    adobe controls whether the input sequence is in Adobe Ascii85 format (i.e.
    is framed with <~ and ~>).

    ignorechars should be a byte string containing characters to ignore from the
    input. This should only contain whitespace characters, and by default
    contains all whitespace characters in ASCII.
364 365

    The result is returned as a bytes object.
366 367 368
    """
    b = _bytes_from_decode_data(b)
    if adobe:
369 370 371 372 373 374 375 376 377
        if not b.endswith(_A85END):
            raise ValueError(
                "Ascii85 encoded byte sequences must end "
                "with {!r}".format(_A85END)
                )
        if b.startswith(_A85START):
            b = b[2:-2]  # Strip off start/end markers
        else:
            b = b[:-2]
378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422
    #
    # We have to go through this stepwise, so as to ignore spaces and handle
    # special short sequences
    #
    packI = struct.Struct('!I').pack
    decoded = []
    decoded_append = decoded.append
    curr = []
    curr_append = curr.append
    curr_clear = curr.clear
    for x in b + b'u' * 4:
        if b'!'[0] <= x <= b'u'[0]:
            curr_append(x)
            if len(curr) == 5:
                acc = 0
                for x in curr:
                    acc = 85 * acc + (x - 33)
                try:
                    decoded_append(packI(acc))
                except struct.error:
                    raise ValueError('Ascii85 overflow') from None
                curr_clear()
        elif x == b'z'[0]:
            if curr:
                raise ValueError('z inside Ascii85 5-tuple')
            decoded_append(b'\0\0\0\0')
        elif foldspaces and x == b'y'[0]:
            if curr:
                raise ValueError('y inside Ascii85 5-tuple')
            decoded_append(b'\x20\x20\x20\x20')
        elif x in ignorechars:
            # Skip whitespace
            continue
        else:
            raise ValueError('Non-Ascii85 digit found: %c' % x)

    result = b''.join(decoded)
    padding = 4 - len(curr)
    if padding:
        # Throw away the extra padding
        result = result[:-padding]
    return result

# The following code is originally taken (with permission) from Mercurial

423 424 425 426
_b85alphabet = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
                b"abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~")
_b85chars = None
_b85chars2 = None
427 428 429
_b85dec = None

def b85encode(b, pad=False):
430
    """Encode bytes-like object b in base85 format and return a bytes object.
431

432 433
    If pad is true, the input is padded with b'\\0' so its length is a multiple of
    4 bytes before encoding.
434
    """
435 436 437 438 439 440
    global _b85chars, _b85chars2
    # Delay the initialization of tables to not waste memory
    # if the function is never called
    if _b85chars is None:
        _b85chars = [bytes((i,)) for i in _b85alphabet]
        _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars]
441 442 443
    return _85encode(b, _b85chars, _b85chars2, pad)

def b85decode(b):
444 445 446 447
    """Decode the base85-encoded bytes-like object or ASCII string b

    The result is returned as a bytes object.
    """
448
    global _b85dec
449 450
    # Delay the initialization of tables to not waste memory
    # if the function is never called
451 452
    if _b85dec is None:
        _b85dec = [None] * 256
453 454
        for i, c in enumerate(_b85alphabet):
            _b85dec[c] = i
455

456
    b = _bytes_from_decode_data(b)
457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482
    padding = (-len(b)) % 5
    b = b + b'~' * padding
    out = []
    packI = struct.Struct('!I').pack
    for i in range(0, len(b), 5):
        chunk = b[i:i + 5]
        acc = 0
        try:
            for c in chunk:
                acc = acc * 85 + _b85dec[c]
        except TypeError:
            for j, c in enumerate(chunk):
                if _b85dec[c] is None:
                    raise ValueError('bad base85 character at position %d'
                                    % (i + j)) from None
            raise
        try:
            out.append(packI(acc))
        except struct.error:
            raise ValueError('base85 overflow in hunk starting at byte %d'
                             % i) from None

    result = b''.join(out)
    if padding:
        result = result[:-padding]
    return result
483

484 485
# Legacy interface.  This code could be cleaned up since I don't believe
# binascii has any line length limitations.  It just doesn't seem worth it
486
# though.  The files should be opened in binary mode.
487

488
MAXLINESIZE = 76 # Excluding the CRLF
489
MAXBINSIZE = (MAXLINESIZE//4)*3
490 491

def encode(input, output):
492
    """Encode a file; input and output are binary files."""
493
    while True:
494
        s = input.read(MAXBINSIZE)
495 496
        if not s:
            break
497 498
        while len(s) < MAXBINSIZE:
            ns = input.read(MAXBINSIZE-len(s))
499 500 501
            if not ns:
                break
            s += ns
502 503
        line = binascii.b2a_base64(s)
        output.write(line)
504

505

506
def decode(input, output):
507
    """Decode a file; input and output are binary files."""
508
    while True:
509
        line = input.readline()
510 511
        if not line:
            break
512 513
        s = binascii.a2b_base64(line)
        output.write(s)
514

515 516 517 518 519 520 521 522 523 524 525 526 527 528 529
def _input_type_check(s):
    try:
        m = memoryview(s)
    except TypeError as err:
        msg = "expected bytes-like object, not %s" % s.__class__.__name__
        raise TypeError(msg) from err
    if m.format not in ('c', 'b', 'B'):
        msg = ("expected single byte elements, not %r from %s" %
                                          (m.format, s.__class__.__name__))
        raise TypeError(msg)
    if m.ndim != 1:
        msg = ("expected 1-D data, not %d-D data from %s" %
                                          (m.ndim, s.__class__.__name__))
        raise TypeError(msg)

530

531
def encodebytes(s):
532
    """Encode a bytestring into a bytes object containing multiple lines
533
    of base-64 data."""
534
    _input_type_check(s)
535 536 537 538
    pieces = []
    for i in range(0, len(s), MAXBINSIZE):
        chunk = s[i : i + MAXBINSIZE]
        pieces.append(binascii.b2a_base64(chunk))
539
    return b"".join(pieces)
540

541 542 543
def encodestring(s):
    """Legacy alias of encodebytes()."""
    import warnings
544 545
    warnings.warn("encodestring() is a deprecated alias since 3.1, "
                  "use encodebytes()",
546 547
                  DeprecationWarning, 2)
    return encodebytes(s)
548

549

550
def decodebytes(s):
551
    """Decode a bytestring of base-64 data into a bytes object."""
552
    _input_type_check(s)
553
    return binascii.a2b_base64(s)
554

555 556 557
def decodestring(s):
    """Legacy alias of decodebytes()."""
    import warnings
558 559
    warnings.warn("decodestring() is a deprecated alias since Python 3.1, "
                  "use decodebytes()",
560 561
                  DeprecationWarning, 2)
    return decodebytes(s)
562

563

564 565 566
# Usable as a script...
def main():
    """Small main program"""
567 568 569
    import sys, getopt
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'deut')
570
    except getopt.error as msg:
571
        sys.stdout = sys.stderr
572 573
        print(msg)
        print("""usage: %s [-d|-e|-u|-t] [file|-]
574 575
        -d, -u: decode
        -e: encode (default)
576
        -t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0])
577 578 579 580 581 582
        sys.exit(2)
    func = encode
    for o, a in opts:
        if o == '-e': func = encode
        if o == '-d': func = decode
        if o == '-u': func = decode
583
        if o == '-t': test(); return
584
    if args and args[0] != '-':
585 586
        with open(args[0], 'rb') as f:
            func(f, sys.stdout.buffer)
587
    else:
588
        func(sys.stdin.buffer, sys.stdout.buffer)
589

590

591 592 593
def test():
    s0 = b"Aladdin:open sesame"
    print(repr(s0))
594
    s1 = encodebytes(s0)
595
    print(repr(s1))
596
    s2 = decodebytes(s1)
597 598
    print(repr(s2))
    assert s0 == s2
599

600

601
if __name__ == '__main__':
602
    main()