Issue #5385: Fixed mmap crash after resize failure on windows.

Now uses NULL instead of INVALID_HANDLE_VALUE as invalid map handle
because CreateFileMapping returns NULL when error occurs.

Lib/test/test_mmap.py

@@ -504,6 +504,7 @@ class MmapTests(unittest.TestCase):
             data1 = "0123456789"
             data2 = "abcdefghij"
             assert len(data1) == len(data2)
+
             # Test same tag
             m1 = mmap.mmap(-1, len(data1), tagname="foo")
             m1[:] = data1
@@ -511,6 +512,9 @@ class MmapTests(unittest.TestCase):
             m2[:] = data2
             self.assertEquals(m1[:], data2)
             self.assertEquals(m2[:], data2)
+            m2.close()
+            m1.close()
+
             # Test differnt tag
             m1 = mmap.mmap(-1, len(data1), tagname="foo")
             m1[:] = data1
@@ -518,14 +522,42 @@ class MmapTests(unittest.TestCase):
             m2[:] = data2
             self.assertEquals(m1[:], data1)
             self.assertEquals(m2[:], data2)
+            m2.close()
+            m1.close()
 
-        def test_tagname_crash(self):
+        def test_crasher_on_windows(self):
             # Should not crash (Issue 1733986)
             m = mmap.mmap(-1, 1000, tagname="foo")
             try:
                 mmap.mmap(-1, 5000, tagname="foo")[:] # same tagname, but larger size
             except:
                 pass
+            m.close()
+
+            # Should not crash (Issue 5385)
+            m = mmap.mmap(-1, 1000)
+            try:
+                m.resize(0)
+            except:
+                pass
+            try:
+                m[:]
+            except:
+                pass
+            m.close()
+
+            m1 = mmap.mmap(-1, 1000)
+            m2 = mmap.mmap(-1, 1000)
+            try:
+                m2.resize(5000)
+            except:
+                pass
+            try:
+                m2[:]
+            except:
+                pass
+            m2.close()
+            m1.close()
 
 
 def test_main():

Misc/NEWS

@@ -168,6 +168,8 @@ Core and Builtins
 Library
 -------
 
+- Issue #5385: Fixed mmap crash after resize failure on windows.
+
 - Issue #5179: Fixed subprocess handle leak on failure on windows.
 
 - PEP 372:  Added collections.OrderedDict().

Modules/mmapmodule.c

@@ -112,7 +112,7 @@ mmap_object_dealloc(mmap_object *m_obj)
 #ifdef MS_WINDOWS
 	if (m_obj->data != NULL)
 		UnmapViewOfFile (m_obj->data);
-	if (m_obj->map_handle != INVALID_HANDLE_VALUE)
+	if (m_obj->map_handle != NULL)
 		CloseHandle (m_obj->map_handle);
 	if (m_obj->file_handle != INVALID_HANDLE_VALUE)
 		CloseHandle (m_obj->file_handle);
@@ -147,9 +147,9 @@ mmap_close_method(mmap_object *self, PyObject *unused)
 		UnmapViewOfFile(self->data);
 		self->data = NULL;
 	}
-	if (self->map_handle != INVALID_HANDLE_VALUE) {
+	if (self->map_handle != NULL) {
 		CloseHandle(self->map_handle);
-		self->map_handle = INVALID_HANDLE_VALUE;
+		self->map_handle = NULL;
 	}
 	if (self->file_handle != INVALID_HANDLE_VALUE) {
 		CloseHandle(self->file_handle);
@@ -173,7 +173,7 @@ mmap_close_method(mmap_object *self, PyObject *unused)
 #ifdef MS_WINDOWS
 #define CHECK_VALID(err)						\
 do {									\
-    if (self->map_handle == INVALID_HANDLE_VALUE) {						\
+    if (self->map_handle == NULL) {					\
 	PyErr_SetString(PyExc_ValueError, "mmap closed or invalid");	\
 	return err;							\
     }									\
@@ -441,8 +441,10 @@ mmap_resize_method(mmap_object *self,
 		DWORD off_hi, off_lo, newSizeLow, newSizeHigh;
 		/* First, unmap the file view */
 		UnmapViewOfFile(self->data);
+		self->data = NULL;
 		/* Close the mapping object */
 		CloseHandle(self->map_handle);
+		self->map_handle = NULL;
 		/* Move to the desired EOF position */
 #if SIZEOF_SIZE_T > 4
 		newSizeHigh = (DWORD)((self->offset + new_size) >> 32);
@@ -479,6 +481,8 @@ mmap_resize_method(mmap_object *self,
 				return Py_None;
 			} else {
 				dwErrCode = GetLastError();
+				CloseHandle(self->map_handle);
+				self->map_handle = NULL;
 			}
 		} else {
 			dwErrCode = GetLastError();
@@ -1279,7 +1283,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
 	   destruct the object in the face of failure */
 	m_obj->data = NULL;
 	m_obj->file_handle = INVALID_HANDLE_VALUE;
-	m_obj->map_handle = INVALID_HANDLE_VALUE;
+	m_obj->map_handle = NULL;
 	m_obj->tagname = NULL;
 	m_obj->offset = offset;
 
@@ -1376,8 +1380,11 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
 						     m_obj->size);
 		if (m_obj->data != NULL)
 			return (PyObject *)m_obj;
-		else
+		else {
 			dwErr = GetLastError();
+			CloseHandle(m_obj->map_handle);
+			m_obj->map_handle = NULL;
+		}
 	} else
 		dwErr = GetLastError();
 	Py_DECREF(m_obj);