Patch #1446489 (zipfile: support for ZIP64)

Doc/lib/libzipfile.tex

@@ -17,7 +17,8 @@ understanding of the format, as defined in
 Note}.
 
 This module does not currently handle ZIP files which have appended
-comments, or multi-disk ZIP files.
+comments, or multi-disk ZIP files. It can handle ZIP files that use the 
+ZIP64 extensions (that is ZIP files that are more than 4 GByte in size).
 
 The available attributes of this module are:
 
@@ -25,6 +26,11 @@ The available attributes of this module are:
   The error raised for bad ZIP files.
 \end{excdesc}
 
+\begin{excdesc}{LargeZipFile}
+  The error raised when a ZIP file would require ZIP64 functionality but that
+  has not been enabled.
+\end{excdesc}
+
 \begin{classdesc*}{ZipFile}
   The class for reading and writing ZIP files.  See
   ``\citetitle{ZipFile Objects}'' (section \ref{zipfile-objects}) for
@@ -77,7 +83,7 @@ The available attributes of this module are:
 
 \subsection{ZipFile Objects \label{zipfile-objects}}
 
-\begin{classdesc}{ZipFile}{file\optional{, mode\optional{, compression}}} 
+\begin{classdesc}{ZipFile}{file\optional{, mode\optional{, compression\optional{, allowZip64}}}} 
   Open a ZIP file, where \var{file} can be either a path to a file
   (a string) or a file-like object.  The \var{mode} parameter
   should be \code{'r'} to read an existing file, \code{'w'} to
@@ -100,6 +106,12 @@ cat myzip.zip >> python.exe
   is specified but the \refmodule{zlib} module is not available,
   \exception{RuntimeError} is also raised.  The default is
   \constant{ZIP_STORED}. 
+  If \var{allowZip64} is \code{True} zipfile will create zipfiles that use
+  the ZIP64 extensions when the zipfile is larger than 2GBytes. If it is 
+  false (the default) zipfile will raise an exception when the zipfile would
+  require ZIP64 extensions. ZIP64 extensions are disabled by default because
+  the default zip and unzip commands on Unix (the InfoZIP utilities) don't 
+  support these extensions.
 \end{classdesc}
 
 \begin{methoddesc}{close}{}
@@ -132,8 +144,8 @@ cat myzip.zip >> python.exe
 \end{methoddesc}
 
 \begin{methoddesc}{testzip}{}
-  Read all the files in the archive and check their CRC's.  Return the
-  name of the first bad file, or else return \code{None}.
+  Read all the files in the archive and check their CRC's and file
+  headers.  Return the name of the first bad file, or else return \code{None}.
 \end{methoddesc}
 
 \begin{methoddesc}{write}{filename\optional{, arcname\optional{,
@@ -284,10 +296,6 @@ Instances have the following attributes:
   Byte offset to the file header.
 \end{memberdesc}
 
-\begin{memberdesc}[ZipInfo]{file_offset}
-  Byte offset to the start of the file data.
-\end{memberdesc}
-
 \begin{memberdesc}[ZipInfo]{CRC}
   CRC-32 of the uncompressed file.
 \end{memberdesc}

Lib/test/test_zipfile.py

@@ -4,7 +4,7 @@ try:
 except ImportError:
     zlib = None
 
-import zipfile, os, unittest
+import zipfile, os, unittest, sys, shutil
 
 from StringIO import StringIO
 from tempfile import TemporaryFile
@@ -28,14 +28,70 @@ class TestsWithSourceFile(unittest.TestCase):
         zipfp = zipfile.ZipFile(f, "w", compression)
         zipfp.write(TESTFN, "another"+os.extsep+"name")
         zipfp.write(TESTFN, TESTFN)
+        zipfp.writestr("strfile", self.data)
         zipfp.close()
 
         # Read the ZIP archive
         zipfp = zipfile.ZipFile(f, "r", compression)
         self.assertEqual(zipfp.read(TESTFN), self.data)
         self.assertEqual(zipfp.read("another"+os.extsep+"name"), self.data)
+        self.assertEqual(zipfp.read("strfile"), self.data)
+
+        # Print the ZIP directory
+        fp = StringIO()
+        stdout = sys.stdout
+        try:
+            sys.stdout = fp
+
+            zipfp.printdir()
+        finally:
+            sys.stdout = stdout
+        
+        directory = fp.getvalue()
+        lines = directory.splitlines()
+        self.assertEquals(len(lines), 4) # Number of files + header
+
+        self.assert_('File Name' in lines[0])
+        self.assert_('Modified' in lines[0])
+        self.assert_('Size' in lines[0])
+
+        fn, date, time, size = lines[1].split()
+        self.assertEquals(fn, 'another.name')
+        # XXX: timestamp is not tested
+        self.assertEquals(size, str(len(self.data)))
+
+        # Check the namelist
+        names = zipfp.namelist()
+        self.assertEquals(len(names), 3)
+        self.assert_(TESTFN in names)
+        self.assert_("another"+os.extsep+"name" in names)
+        self.assert_("strfile" in names)
+
+        # Check infolist
+        infos = zipfp.infolist()
+        names = [ i.filename for i in infos ]
+        self.assertEquals(len(names), 3)
+        self.assert_(TESTFN in names)
+        self.assert_("another"+os.extsep+"name" in names)
+        self.assert_("strfile" in names)
+        for i in infos:
+            self.assertEquals(i.file_size, len(self.data))
+
+        # check getinfo
+        for nm in (TESTFN, "another"+os.extsep+"name", "strfile"):
+            info = zipfp.getinfo(nm)
+            self.assertEquals(info.filename, nm)
+            self.assertEquals(info.file_size, len(self.data))
+
+        # Check that testzip doesn't raise an exception
+        zipfp.testzip()
+
+
         zipfp.close()
 
+
+
+
     def testStored(self):
         for f in (TESTFN2, TemporaryFile(), StringIO()):
             self.zipTest(f, zipfile.ZIP_STORED)
@@ -59,6 +115,197 @@ class TestsWithSourceFile(unittest.TestCase):
         os.remove(TESTFN)
         os.remove(TESTFN2)
 
+class TestZip64InSmallFiles(unittest.TestCase):
+    # These tests test the ZIP64 functionality without using large files,
+    # see test_zipfile64 for proper tests.
+
+    def setUp(self):
+        self._limit = zipfile.ZIP64_LIMIT
+        zipfile.ZIP64_LIMIT = 5
+
+        line_gen = ("Test of zipfile line %d." % i for i in range(0, 1000))
+        self.data = '\n'.join(line_gen)
+
+        # Make a source file with some lines
+        fp = open(TESTFN, "wb")
+        fp.write(self.data)
+        fp.close()
+
+    def largeFileExceptionTest(self, f, compression):
+        zipfp = zipfile.ZipFile(f, "w", compression)
+        self.assertRaises(zipfile.LargeZipFile, 
+                zipfp.write, TESTFN, "another"+os.extsep+"name")
+        zipfp.close()
+
+    def largeFileExceptionTest2(self, f, compression):
+        zipfp = zipfile.ZipFile(f, "w", compression)
+        self.assertRaises(zipfile.LargeZipFile, 
+                zipfp.writestr, "another"+os.extsep+"name", self.data)
+        zipfp.close()
+
+    def testLargeFileException(self):
+        for f in (TESTFN2, TemporaryFile(), StringIO()):
+            self.largeFileExceptionTest(f, zipfile.ZIP_STORED)
+            self.largeFileExceptionTest2(f, zipfile.ZIP_STORED)
+
+    def zipTest(self, f, compression):
+        # Create the ZIP archive
+        zipfp = zipfile.ZipFile(f, "w", compression, allowZip64=True)
+        zipfp.write(TESTFN, "another"+os.extsep+"name")
+        zipfp.write(TESTFN, TESTFN)
+        zipfp.writestr("strfile", self.data)
+        zipfp.close()
+
+        # Read the ZIP archive
+        zipfp = zipfile.ZipFile(f, "r", compression)
+        self.assertEqual(zipfp.read(TESTFN), self.data)
+        self.assertEqual(zipfp.read("another"+os.extsep+"name"), self.data)
+        self.assertEqual(zipfp.read("strfile"), self.data)
+
+        # Print the ZIP directory
+        fp = StringIO()
+        stdout = sys.stdout
+        try:
+            sys.stdout = fp
+
+            zipfp.printdir()
+        finally:
+            sys.stdout = stdout
+        
+        directory = fp.getvalue()
+        lines = directory.splitlines()
+        self.assertEquals(len(lines), 4) # Number of files + header
+
+        self.assert_('File Name' in lines[0])
+        self.assert_('Modified' in lines[0])
+        self.assert_('Size' in lines[0])
+
+        fn, date, time, size = lines[1].split()
+        self.assertEquals(fn, 'another.name')
+        # XXX: timestamp is not tested
+        self.assertEquals(size, str(len(self.data)))
+
+        # Check the namelist
+        names = zipfp.namelist()
+        self.assertEquals(len(names), 3)
+        self.assert_(TESTFN in names)
+        self.assert_("another"+os.extsep+"name" in names)
+        self.assert_("strfile" in names)
+
+        # Check infolist
+        infos = zipfp.infolist()
+        names = [ i.filename for i in infos ]
+        self.assertEquals(len(names), 3)
+        self.assert_(TESTFN in names)
+        self.assert_("another"+os.extsep+"name" in names)
+        self.assert_("strfile" in names)
+        for i in infos:
+            self.assertEquals(i.file_size, len(self.data))
+
+        # check getinfo
+        for nm in (TESTFN, "another"+os.extsep+"name", "strfile"):
+            info = zipfp.getinfo(nm)
+            self.assertEquals(info.filename, nm)
+            self.assertEquals(info.file_size, len(self.data))
+
+        # Check that testzip doesn't raise an exception
+        zipfp.testzip()
+
+
+        zipfp.close()
+
+    def testStored(self):
+        for f in (TESTFN2, TemporaryFile(), StringIO()):
+            self.zipTest(f, zipfile.ZIP_STORED)
+
+
+    if zlib:
+        def testDeflated(self):
+            for f in (TESTFN2, TemporaryFile(), StringIO()):
+                self.zipTest(f, zipfile.ZIP_DEFLATED)
+
+    def testAbsoluteArcnames(self):
+        zipfp = zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED, allowZip64=True)
+        zipfp.write(TESTFN, "/absolute")
+        zipfp.close()
+
+        zipfp = zipfile.ZipFile(TESTFN2, "r", zipfile.ZIP_STORED)
+        self.assertEqual(zipfp.namelist(), ["absolute"])
+        zipfp.close()
+
+
+    def tearDown(self):
+        zipfile.ZIP64_LIMIT = self._limit
+        os.remove(TESTFN)
+        os.remove(TESTFN2)
+
+class PyZipFileTests(unittest.TestCase):
+    def testWritePyfile(self):
+        zipfp  = zipfile.PyZipFile(TemporaryFile(), "w")
+        fn = __file__
+        if fn.endswith('.pyc') or fn.endswith('.pyo'):
+            fn = fn[:-1]
+
+        zipfp.writepy(fn)
+
+        bn = os.path.basename(fn)
+        self.assert_(bn not in zipfp.namelist())
+        self.assert_(bn + 'o' in zipfp.namelist() or bn + 'c' in zipfp.namelist())
+        zipfp.close()
+
+
+        zipfp  = zipfile.PyZipFile(TemporaryFile(), "w")
+        fn = __file__
+        if fn.endswith('.pyc') or fn.endswith('.pyo'):
+            fn = fn[:-1]
+
+        zipfp.writepy(fn, "testpackage")
+
+        bn = "%s/%s"%("testpackage", os.path.basename(fn))
+        self.assert_(bn not in zipfp.namelist())
+        self.assert_(bn + 'o' in zipfp.namelist() or bn + 'c' in zipfp.namelist())
+        zipfp.close()
+
+    def testWritePythonPackage(self):
+        import email
+        packagedir = os.path.dirname(email.__file__)
+
+        zipfp  = zipfile.PyZipFile(TemporaryFile(), "w")
+        zipfp.writepy(packagedir)
+
+        # Check for a couple of modules at different levels of the hieararchy
+        names = zipfp.namelist()
+        self.assert_('email/__init__.pyo' in names or 'email/__init__.pyc' in names)
+        self.assert_('email/mime/text.pyo' in names or 'email/mime/text.pyc' in names)
+
+    def testWritePythonDirectory(self):
+        os.mkdir(TESTFN2)
+        try:
+            fp = open(os.path.join(TESTFN2, "mod1.py"), "w")
+            fp.write("print 42\n")
+            fp.close()
+
+            fp = open(os.path.join(TESTFN2, "mod2.py"), "w")
+            fp.write("print 42 * 42\n")
+            fp.close()
+
+            fp = open(os.path.join(TESTFN2, "mod2.txt"), "w")
+            fp.write("bla bla bla\n")
+            fp.close()
+
+            zipfp  = zipfile.PyZipFile(TemporaryFile(), "w")
+            zipfp.writepy(TESTFN2)
+
+            names = zipfp.namelist()
+            self.assert_('mod1.pyc' in names or 'mod1.pyo' in names)
+            self.assert_('mod2.pyc' in names or 'mod2.pyo' in names)
+            self.assert_('mod2.txt' not in names)
+
+        finally:
+            shutil.rmtree(TESTFN2)
+
+
+
 class OtherTests(unittest.TestCase):
     def testCloseErroneousFile(self):
         # This test checks that the ZipFile constructor closes the file object
@@ -103,7 +350,8 @@ class OtherTests(unittest.TestCase):
         self.assertRaises(RuntimeError, zipf.testzip)
 
 def test_main():
-    run_unittest(TestsWithSourceFile, OtherTests)
+    run_unittest(TestsWithSourceFile, TestZip64InSmallFiles, OtherTests, PyZipFileTests)
+    #run_unittest(TestZip64InSmallFiles)
 
 if __name__ == "__main__":
     test_main()

Lib/test/test_zipfile64.py

+# Tests of the full ZIP64 functionality of zipfile
+# The test_support.requires call is the only reason for keeping this separate
+# from test_zipfile
+from test import test_support
+test_support.requires(
+        'largefile', 
+        'test requires loads of disk-space bytes and a long time to run'
+    )
+
+# We can test part of the module without zlib.
+try:
+    import zlib
+except ImportError:
+    zlib = None
+
+import zipfile, os, unittest
+
+from StringIO import StringIO
+from tempfile import TemporaryFile
+
+from test.test_support import TESTFN, run_unittest
+
+TESTFN2 = TESTFN + "2"
+
+class TestsWithSourceFile(unittest.TestCase):
+    def setUp(self):
+        line_gen = ("Test of zipfile line %d." % i for i in range(0, 1000000))
+        self.data = '\n'.join(line_gen)
+
+        # Make a source file with some lines
+        fp = open(TESTFN, "wb")
+        fp.write(self.data)
+        fp.close()
+
+    def zipTest(self, f, compression):
+        # Create the ZIP archive
+        filecount = int(((1 << 32) / len(self.data)) * 1.5)
+        zipfp = zipfile.ZipFile(f, "w", compression, allowZip64=True)
+
+        for num in range(filecount):
+            zipfp.writestr("testfn%d"%(num,), self.data)
+        zipfp.close()
+
+        # Read the ZIP archive
+        zipfp = zipfile.ZipFile(f, "r", compression)
+        for num in range(filecount):
+            self.assertEqual(zipfp.read("testfn%d"%(num,)), self.data)
+        zipfp.close()
+
+    def testStored(self):
+        for f in (TESTFN2, TemporaryFile()):
+            self.zipTest(f, zipfile.ZIP_STORED)
+
+    if zlib:
+        def testDeflated(self):
+            for f in (TESTFN2, TemporaryFile()):
+                self.zipTest(f, zipfile.ZIP_DEFLATED)
+
+    def tearDown(self):
+        os.remove(TESTFN)
+        os.remove(TESTFN2)
+
+def test_main():
+    run_unittest(TestsWithSourceFile)
+
+if __name__ == "__main__":
+    test_main()

Misc/NEWS

@@ -152,6 +152,7 @@ Extension Modules
   aborts the db transaction safely when a modifier callback fails.
   Fixes SF python patch/bug #1408584.
 
+- Patch #1446489: add support for the ZIP64 extensions to zipfile. 
 
 Library
 -------