Issue #14455: Fix some issues with plistlib

* Negative integer support in binary plists was broken

* Better exception for invalid data

* Fix the versionadded/versionchanged markup in the documentation

* Add the interface cleanup to what's new for 3.4

Doc/library/plistlib.rst

@@ -79,6 +79,8 @@ This module defines the following functions:
    Load a plist from a bytes object. See :func:`load` for an explanation of
    the keyword arguments.
 
+   .. versionadded:: 3.4
+
 
 .. function:: dump(value, fp, \*, fmt=FMT_XML, sort_keys=True, skipkeys=False)
 
@@ -102,8 +104,17 @@ This module defines the following functions:
    A :exc:`TypeError` will be raised if the object is of an unsupported type or
    a container that contains objects of unsupported types.
 
-   .. versionchanged:: 3.4
-      Added the *fmt*, *sort_keys* and *skipkeys* arguments.
+   An :exc:`OverflowError` will be raised for integer values that cannot
+   be represented in (binary) plist files.
+
+   .. warning::
+
+      For compatibility with Apple's libraries it is possible to write
+      an integer in the range from 2 ** 63 upto (and including) 2 ** 64
+      to binary plists, even though these will be read back as negative
+      values.
+
+   .. versionadded: 3.4
 
 
 .. function:: dumps(value, \*, fmt=FMT_XML, sort_keys=True, skipkeys=False)
@@ -112,6 +123,7 @@ This module defines the following functions:
    the documentation for :func:`dump` for an explanation of the keyword
    arguments of this function.
 
+   .. versionadded: 3.4
 
 The following functions are deprecated:
 
@@ -162,9 +174,6 @@ The following functions are deprecated:
 
    .. deprecated:: 3.4 Use :func:`dumps` instead.
 
-   .. versionchanged:: 3.4
-      Added the *fmt*, *sort_keys* and *skipkeys* arguments.
-
 
 The following classes are available:
 

Doc/whatsnew/3.4.rst

@@ -132,6 +132,8 @@ Significantly Improved Library Modules:
   a new :mod:`~email.message.Message` subclass
   (:class:`~email.contentmanager.EmailMessage`) that :ref:`simplify MIME
   handling <whatsnew_email_contentmanager>` (:issue:`18891`).
+* :mod:`plistlib` has a cleaned up interface and support for binary
+  plist files (:issue:`14455`)
 
 CPython implementation improvements:
 

Lib/plistlib.py

@@ -478,7 +478,10 @@ class _PlistWriter(_DumbXMLWriter):
             self.simple_element("false")
 
         elif isinstance(value, int):
-            self.simple_element("integer", "%d" % value)
+            if -1 << 63 <= value < 1 << 64:
+                self.simple_element("integer", "%d" % value)
+            else:
+                raise OverflowError(value)
 
         elif isinstance(value, float):
             self.simple_element("real", repr(value))
@@ -665,7 +668,8 @@ class _BinaryPlistParser:
             return b''
 
         elif tokenH == 0x10:  # int
-            return int.from_bytes(self._fp.read(1 << tokenL), 'big')
+            return int.from_bytes(self._fp.read(1 << tokenL),
+                                  'big', signed=tokenL >= 3)
 
         elif token == 0x22: # real
             return struct.unpack('>f', self._fp.read(4))[0]
@@ -871,14 +875,22 @@ class _BinaryPlistWriter (object):
             self._fp.write(b'\x09')
 
         elif isinstance(value, int):
-            if value < 1 << 8:
+            if value < 0:
+                try:
+                    self._fp.write(struct.pack('>Bq', 0x13, value))
+                except struct.error:
+                    raise OverflowError(value)
+            elif value < 1 << 8:
                 self._fp.write(struct.pack('>BB', 0x10, value))
             elif value < 1 << 16:
                 self._fp.write(struct.pack('>BH', 0x11, value))
             elif value < 1 << 32:
                 self._fp.write(struct.pack('>BL', 0x12, value))
             else:
-                self._fp.write(struct.pack('>BQ', 0x13, value))
+                try:
+                    self._fp.write(struct.pack('>BQ', 0x13, value))
+                except struct.error:
+                    raise OverflowError(value)
 
         elif isinstance(value, float):
             self._fp.write(struct.pack('>Bd', 0x23, value))
@@ -933,7 +945,7 @@ class _BinaryPlistWriter (object):
             self._fp.write(struct.pack('>' + self._ref_format * s, *valRefs))
 
         else:
-            raise InvalidFileException()
+            raise TypeError(value)
 
 
 def _is_fmt_binary(header):

Mac/Tools/plistlib_generate_testdata.py

@@ -23,7 +23,13 @@ def nsstr(value):
 def main():
     pl = OrderedDict()
 
+    # Note: pl is an OrderedDict to control the order
+    # of keys, and hence have some control on the structure
+    # of the output file.
+    # New keys should be added in alphabetical order.
+
     seconds = datetime.datetime(2004, 10, 26, 10, 33, 33, tzinfo=datetime.timezone(datetime.timedelta(0))).timestamp()
+    pl[nsstr('aBigInt')] = 2 ** 63 - 44
     pl[nsstr('aDate')] = NSDate.dateWithTimeIntervalSince1970_(seconds)
 
     pl[nsstr('aDict')] = d = OrderedDict()
@@ -52,6 +58,8 @@ def main():
     aa.append(2)
     aa.append(3)
 
+    pl[nsstr('aNegativeBigInt')] = -80000000000
+    pl[nsstr('aNegativeInt')] = -5
     pl[nsstr('aString')] = nsstr('Doodah')
 
     pl[nsstr('anEmptyDict')] = NSMutableDictionary.alloc().init()

Misc/NEWS

@@ -72,6 +72,8 @@ Library
 
 - Issue #20229: Avoid plistlib deprecation warning in platform.mac_ver().
 
+- Issue #14455: Fix some problems with the new binary plist support in plistlib.
+
 IDLE
 ----