Issue #11750: The Windows API functions scattered in the _subprocess and

_multiprocessing.win32 modules now live in a single module "_winapi".
Patch by sbt.

Lib/multiprocessing/forking.py

@@ -181,10 +181,9 @@ if sys.platform != 'win32':
 else:
     import _thread
     import msvcrt
-    import _subprocess
+    import _winapi
 
     from pickle import load, HIGHEST_PROTOCOL
-    from _multiprocessing import win32
 
     def dump(obj, file, protocol=None):
         ForkingPickler(file, protocol).dump(obj)
@@ -197,8 +196,8 @@ else:
     WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
     WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
 
-    exit = win32.ExitProcess
-    close = win32.CloseHandle
+    exit = _winapi.ExitProcess
+    close = _winapi.CloseHandle
 
     #
     # _python_exe is the assumed path to the python executable.
@@ -220,11 +219,11 @@ else:
 
     def duplicate(handle, target_process=None, inheritable=False):
         if target_process is None:
-            target_process = _subprocess.GetCurrentProcess()
-        return _subprocess.DuplicateHandle(
-            _subprocess.GetCurrentProcess(), handle, target_process,
-            0, inheritable, _subprocess.DUPLICATE_SAME_ACCESS
-            ).Detach()
+            target_process = _winapi.GetCurrentProcess()
+        return _winapi.DuplicateHandle(
+            _winapi.GetCurrentProcess(), handle, target_process,
+            0, inheritable, _winapi.DUPLICATE_SAME_ACCESS
+            )
 
     #
     # We define a Popen class similar to the one from subprocess, but
@@ -248,10 +247,10 @@ else:
             # start process
             cmd = get_command_line() + [rhandle]
             cmd = ' '.join('"%s"' % x for x in cmd)
-            hp, ht, pid, tid = _subprocess.CreateProcess(
+            hp, ht, pid, tid = _winapi.CreateProcess(
                 _python_exe, cmd, None, None, 1, 0, None, None, None
                 )
-            ht.Close()
+            _winapi.CloseHandle(ht)
             close(rhandle)
 
             # set attributes of self
@@ -282,13 +281,13 @@ else:
         def wait(self, timeout=None):
             if self.returncode is None:
                 if timeout is None:
-                    msecs = _subprocess.INFINITE
+                    msecs = _winapi.INFINITE
                 else:
                     msecs = max(0, int(timeout * 1000 + 0.5))
 
-                res = _subprocess.WaitForSingleObject(int(self._handle), msecs)
-                if res == _subprocess.WAIT_OBJECT_0:
-                    code = _subprocess.GetExitCodeProcess(self._handle)
+                res = _winapi.WaitForSingleObject(int(self._handle), msecs)
+                if res == _winapi.WAIT_OBJECT_0:
+                    code = _winapi.GetExitCodeProcess(self._handle)
                     if code == TERMINATE:
                         code = -signal.SIGTERM
                     self.returncode = code
@@ -301,7 +300,7 @@ else:
         def terminate(self):
             if self.returncode is None:
                 try:
-                    _subprocess.TerminateProcess(int(self._handle), TERMINATE)
+                    _winapi.TerminateProcess(int(self._handle), TERMINATE)
                 except WindowsError:
                     if self.wait(timeout=0.1) is None:
                         raise

Lib/multiprocessing/heap.py

@@ -51,7 +51,7 @@ __all__ = ['BufferWrapper']
 
 if sys.platform == 'win32':
 
-    from _multiprocessing import win32
+    import _winapi
 
     class Arena(object):
 
@@ -61,7 +61,7 @@ if sys.platform == 'win32':
             self.size = size
             self.name = 'pym-%d-%d' % (os.getpid(), next(Arena._counter))
             self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
-            assert win32.GetLastError() == 0, 'tagname already in use'
+            assert _winapi.GetLastError() == 0, 'tagname already in use'
             self._state = (self.size, self.name)
 
         def __getstate__(self):
@@ -71,7 +71,7 @@ if sys.platform == 'win32':
         def __setstate__(self, state):
             self.size, self.name = self._state = state
             self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
-            assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
+            assert _winapi.GetLastError() == _winapi.ERROR_ALREADY_EXISTS
 
 else:
 

Lib/multiprocessing/reduction.py

@@ -60,11 +60,11 @@ if not(sys.platform == 'win32' or (hasattr(socket, 'CMSG_LEN') and
 #
 
 if sys.platform == 'win32':
-    from _multiprocessing import win32
+    import _winapi
 
     def send_handle(conn, handle, destination_pid):
-        process_handle = win32.OpenProcess(
-            win32.PROCESS_ALL_ACCESS, False, destination_pid
+        process_handle = _winapi.OpenProcess(
+            _winapi.PROCESS_ALL_ACCESS, False, destination_pid
             )
         try:
             new_handle = duplicate(handle, process_handle)

Lib/subprocess.py

@@ -385,7 +385,7 @@ class TimeoutExpired(SubprocessError):
 if mswindows:
     import threading
     import msvcrt
-    import _subprocess
+    import _winapi
     class STARTUPINFO:
         dwFlags = 0
         hStdInput = None
@@ -410,15 +410,36 @@ __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput",
            "getoutput", "check_output", "CalledProcessError", "DEVNULL"]
 
 if mswindows:
-    from _subprocess import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP,
-                             STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
-                             STD_ERROR_HANDLE, SW_HIDE,
-                             STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW)
+    from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP,
+                         STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
+                         STD_ERROR_HANDLE, SW_HIDE,
+                         STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW)
 
     __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP",
                     "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE",
                     "STD_ERROR_HANDLE", "SW_HIDE",
                     "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"])
+
+    class Handle(int):
+        closed = False
+
+        def Close(self, CloseHandle=_winapi.CloseHandle):
+            if not self.closed:
+                self.closed = True
+                CloseHandle(self)
+
+        def Detach(self):
+            if not self.closed:
+                self.closed = True
+                return int(self)
+            raise ValueError("already closed")
+
+        def __repr__(self):
+            return "Handle(%d)" % int(self)
+
+        __del__ = Close
+        __str__ = __repr__
+
 try:
     MAXFD = os.sysconf("SC_OPEN_MAX")
 except:
@@ -892,11 +913,14 @@ class Popen(object):
             errread, errwrite = -1, -1
 
             if stdin is None:
-                p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE)
+                p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)
                 if p2cread is None:
-                    p2cread, _ = _subprocess.CreatePipe(None, 0)
+                    p2cread, _ = _winapi.CreatePipe(None, 0)
+                    p2cread = Handle(p2cread)
+                    _winapi.CloseHandle(_)
             elif stdin == PIPE:
-                p2cread, p2cwrite = _subprocess.CreatePipe(None, 0)
+                p2cread, p2cwrite = _winapi.CreatePipe(None, 0)
+                p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
             elif stdin == DEVNULL:
                 p2cread = msvcrt.get_osfhandle(self._get_devnull())
             elif isinstance(stdin, int):
@@ -907,11 +931,14 @@ class Popen(object):
             p2cread = self._make_inheritable(p2cread)
 
             if stdout is None:
-                c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE)
+                c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
                 if c2pwrite is None:
-                    _, c2pwrite = _subprocess.CreatePipe(None, 0)
+                    _, c2pwrite = _winapi.CreatePipe(None, 0)
+                    c2pwrite = Handle(c2pwrite)
+                    _winapi.CloseHandle(_)
             elif stdout == PIPE:
-                c2pread, c2pwrite = _subprocess.CreatePipe(None, 0)
+                c2pread, c2pwrite = _winapi.CreatePipe(None, 0)
+                c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
             elif stdout == DEVNULL:
                 c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
             elif isinstance(stdout, int):
@@ -922,11 +949,14 @@ class Popen(object):
             c2pwrite = self._make_inheritable(c2pwrite)
 
             if stderr is None:
-                errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE)
+                errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
                 if errwrite is None:
-                    _, errwrite = _subprocess.CreatePipe(None, 0)
+                    _, errwrite = _winapi.CreatePipe(None, 0)
+                    errwrite = Handle(errwrite)
+                    _winapi.CloseHandle(_)
             elif stderr == PIPE:
-                errread, errwrite = _subprocess.CreatePipe(None, 0)
+                errread, errwrite = _winapi.CreatePipe(None, 0)
+                errread, errwrite = Handle(errread), Handle(errwrite)
             elif stderr == STDOUT:
                 errwrite = c2pwrite
             elif stderr == DEVNULL:
@@ -945,15 +975,17 @@ class Popen(object):
 
         def _make_inheritable(self, handle):
             """Return a duplicate of handle, which is inheritable"""
-            return _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(),
-                                handle, _subprocess.GetCurrentProcess(), 0, 1,
-                                _subprocess.DUPLICATE_SAME_ACCESS)
+            h = _winapi.DuplicateHandle(
+                _winapi.GetCurrentProcess(), handle,
+                _winapi.GetCurrentProcess(), 0, 1,
+                _winapi.DUPLICATE_SAME_ACCESS)
+            return Handle(h)
 
 
         def _find_w9xpopen(self):
             """Find and return absolut path to w9xpopen.exe"""
             w9xpopen = os.path.join(
-                            os.path.dirname(_subprocess.GetModuleFileName(0)),
+                            os.path.dirname(_winapi.GetModuleFileName(0)),
                                     "w9xpopen.exe")
             if not os.path.exists(w9xpopen):
                 # Eeek - file-not-found - possibly an embedding
@@ -985,17 +1017,17 @@ class Popen(object):
             if startupinfo is None:
                 startupinfo = STARTUPINFO()
             if -1 not in (p2cread, c2pwrite, errwrite):
-                startupinfo.dwFlags |= _subprocess.STARTF_USESTDHANDLES
+                startupinfo.dwFlags |= _winapi.STARTF_USESTDHANDLES
                 startupinfo.hStdInput = p2cread
                 startupinfo.hStdOutput = c2pwrite
                 startupinfo.hStdError = errwrite
 
             if shell:
-                startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW
-                startupinfo.wShowWindow = _subprocess.SW_HIDE
+                startupinfo.dwFlags |= _winapi.STARTF_USESHOWWINDOW
+                startupinfo.wShowWindow = _winapi.SW_HIDE
                 comspec = os.environ.get("COMSPEC", "cmd.exe")
                 args = '{} /c "{}"'.format (comspec, args)
-                if (_subprocess.GetVersion() >= 0x80000000 or
+                if (_winapi.GetVersion() >= 0x80000000 or
                         os.path.basename(comspec).lower() == "command.com"):
                     # Win9x, or using command.com on NT. We need to
                     # use the w9xpopen intermediate program. For more
@@ -1009,11 +1041,11 @@ class Popen(object):
                     # use at xxx" and a hopeful warning about the
                     # stability of your system.  Cost is Ctrl+C won't
                     # kill children.
-                    creationflags |= _subprocess.CREATE_NEW_CONSOLE
+                    creationflags |= _winapi.CREATE_NEW_CONSOLE
 
             # Start the process
             try:
-                hp, ht, pid, tid = _subprocess.CreateProcess(executable, args,
+                hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
                                          # no special security
                                          None, None,
                                          int(not close_fds),
@@ -1045,14 +1077,14 @@ class Popen(object):
 
             # Retain the process handle, but close the thread handle
             self._child_created = True
-            self._handle = hp
+            self._handle = Handle(hp)
             self.pid = pid
-            ht.Close()
+            _winapi.CloseHandle(ht)
 
         def _internal_poll(self, _deadstate=None,
-                _WaitForSingleObject=_subprocess.WaitForSingleObject,
-                _WAIT_OBJECT_0=_subprocess.WAIT_OBJECT_0,
-                _GetExitCodeProcess=_subprocess.GetExitCodeProcess):
+                _WaitForSingleObject=_winapi.WaitForSingleObject,
+                _WAIT_OBJECT_0=_winapi.WAIT_OBJECT_0,
+                _GetExitCodeProcess=_winapi.GetExitCodeProcess):
             """Check if child process has terminated.  Returns returncode
             attribute.
 
@@ -1072,15 +1104,15 @@ class Popen(object):
             if endtime is not None:
                 timeout = self._remaining_time(endtime)
             if timeout is None:
-                timeout_millis = _subprocess.INFINITE
+                timeout_millis = _winapi.INFINITE
             else:
                 timeout_millis = int(timeout * 1000)
             if self.returncode is None:
-                result = _subprocess.WaitForSingleObject(self._handle,
-                                                         timeout_millis)
-                if result == _subprocess.WAIT_TIMEOUT:
+                result = _winapi.WaitForSingleObject(self._handle,
+                                                    timeout_millis)
+                if result == _winapi.WAIT_TIMEOUT:
                     raise TimeoutExpired(self.args, timeout)
-                self.returncode = _subprocess.GetExitCodeProcess(self._handle)
+                self.returncode = _winapi.GetExitCodeProcess(self._handle)
             return self.returncode
 
 
@@ -1163,12 +1195,12 @@ class Popen(object):
             """Terminates the process
             """
             try:
-                _subprocess.TerminateProcess(self._handle, 1)
+                _winapi.TerminateProcess(self._handle, 1)
             except PermissionError:
                 # ERROR_ACCESS_DENIED (winerror 5) is received when the
                 # process already died.
-                rc = _subprocess.GetExitCodeProcess(self._handle)
-                if rc == _subprocess.STILL_ACTIVE:
+                rc = _winapi.GetExitCodeProcess(self._handle)
+                if rc == _winapi.STILL_ACTIVE:
                     raise
                 self.returncode = rc
 

Lib/test/test_multiprocessing.py

@@ -84,7 +84,7 @@ HAVE_GETVALUE = not getattr(_multiprocessing,
 
 WIN32 = (sys.platform == "win32")
 if WIN32:
-    from _subprocess import WaitForSingleObject, INFINITE, WAIT_OBJECT_0
+    from _winapi import WaitForSingleObject, INFINITE, WAIT_OBJECT_0
 
     def wait_for_handle(handle, timeout):
         if timeout is None or timeout < 0.0:

Misc/NEWS

@@ -47,6 +47,10 @@ Core and Builtins
 Library
 -------
 
+- Issue #11750: The Windows API functions scattered in the _subprocess and
+  _multiprocessing.win32 modules now live in a single module "_winapi".
+  Patch by sbt.
+
 - Issue #14087: multiprocessing: add Condition.wait_for(). Patch by sbt.
 
 - Issue #14452: SysLogHandler no longer inserts a UTF-8 BOM into the message.

Modules/_multiprocessing/multiprocessing.c

@@ -9,8 +9,6 @@
 #include "multiprocessing.h"
 
 
-PyObject *create_win32_namespace(void);
-
 PyObject *ProcessError, *BufferTooShort;
 
 /*
@@ -66,6 +64,72 @@ multiprocessing_address_of_buffer(PyObject *self, PyObject *obj)
                          PyLong_FromVoidPtr(buffer), buffer_len);
 }
 
+#ifdef MS_WINDOWS
+static PyObject *
+multiprocessing_closesocket(PyObject *self, PyObject *args)
+{
+    HANDLE handle;
+    int ret;
+
+    if (!PyArg_ParseTuple(args, F_HANDLE ":closesocket" , &handle))
+        return NULL;
+
+    Py_BEGIN_ALLOW_THREADS
+    ret = closesocket((SOCKET) handle);
+    Py_END_ALLOW_THREADS
+
+    if (ret)
+        return PyErr_SetExcFromWindowsErr(PyExc_IOError, WSAGetLastError());
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+multiprocessing_recv(PyObject *self, PyObject *args)
+{
+    HANDLE handle;
+    int size, nread;
+    PyObject *buf;
+
+    if (!PyArg_ParseTuple(args, F_HANDLE "i:recv" , &handle, &size))
+        return NULL;
+
+    buf = PyBytes_FromStringAndSize(NULL, size);
+    if (!buf)
+        return NULL;
+
+    Py_BEGIN_ALLOW_THREADS
+    nread = recv((SOCKET) handle, PyBytes_AS_STRING(buf), size, 0);
+    Py_END_ALLOW_THREADS
+
+    if (nread < 0) {
+        Py_DECREF(buf);
+        return PyErr_SetExcFromWindowsErr(PyExc_IOError, WSAGetLastError());
+    }
+    _PyBytes_Resize(&buf, nread);
+    return buf;
+}
+
+static PyObject *
+multiprocessing_send(PyObject *self, PyObject *args)
+{
+    HANDLE handle;
+    Py_buffer buf;
+    int ret;
+
+    if (!PyArg_ParseTuple(args, F_HANDLE "y*:send" , &handle, &buf))
+        return NULL;
+
+    Py_BEGIN_ALLOW_THREADS
+    ret = send((SOCKET) handle, buf.buf, buf.len, 0);
+    Py_END_ALLOW_THREADS
+
+    PyBuffer_Release(&buf);
+    if (ret < 0)
+        return PyErr_SetExcFromWindowsErr(PyExc_IOError, WSAGetLastError());
+    return PyLong_FromLong(ret);
+}
+
+#endif
 
 /*
  * Function table
@@ -75,6 +139,11 @@ static PyMethodDef module_methods[] = {
     {"address_of_buffer", multiprocessing_address_of_buffer, METH_O,
      "address_of_buffer(obj) -> int\n"
      "Return address of obj assuming obj supports buffer inteface"},
+#ifdef MS_WINDOWS
+    {"closesocket", multiprocessing_closesocket, METH_VARARGS, ""},
+    {"recv", multiprocessing_recv, METH_VARARGS, ""},
+    {"send", multiprocessing_send, METH_VARARGS, ""},
+#endif
     {NULL}
 };
 
@@ -135,14 +204,6 @@ PyInit__multiprocessing(void)
     PyModule_AddObject(module, "SemLock", (PyObject*)&SemLockType);
 #endif
 
-#ifdef MS_WINDOWS
-    /* Initialize win32 class and add to multiprocessing */
-    temp = create_win32_namespace();
-    if (!temp)
-        return NULL;
-    PyModule_AddObject(module, "win32", temp);
-#endif
-
     /* Add configuration macros */
     temp = PyDict_New();
     if (!temp)
@@ -152,7 +213,7 @@ PyInit__multiprocessing(void)
     value = Py_BuildValue("i", name);                             \
     if (value == NULL) { Py_DECREF(temp); return NULL; }          \
     if (PyDict_SetItemString(temp, #name, value) < 0) {           \
-        Py_DECREF(temp); Py_DECREF(value); return NULL; }                 \
+        Py_DECREF(temp); Py_DECREF(value); return NULL; }         \
     Py_DECREF(value)
 
 #if defined(HAVE_SEM_OPEN) && !defined(POSIX_SEMAPHORES_NOT_ENABLED)

PC/config.c

@@ -56,7 +56,7 @@ extern PyObject* PyInit__codecs_iso2022(void);
 extern PyObject* PyInit__codecs_jp(void);
 extern PyObject* PyInit__codecs_kr(void);
 extern PyObject* PyInit__codecs_tw(void);
-extern PyObject* PyInit__subprocess(void);
+extern PyObject* PyInit__winapi(void);
 extern PyObject* PyInit__lsprof(void);
 extern PyObject* PyInit__ast(void);
 extern PyObject* PyInit__io(void);
@@ -101,8 +101,8 @@ struct _inittab _PyImport_Inittab[] = {
     {"msvcrt", PyInit_msvcrt},
     {"_locale", PyInit__locale},
 #endif
-    /* XXX Should _subprocess go in a WIN32 block?  not WIN64? */
-    {"_subprocess", PyInit__subprocess},
+    /* XXX Should _winapi go in a WIN32 block?  not WIN64? */
+    {"_winapi", PyInit__winapi},
 
     {"_codecs", PyInit__codecs},
     {"_weakref", PyInit__weakref},

PCbuild/_multiprocessing.vcproj

@@ -534,10 +534,6 @@
 				RelativePath="..\Modules\_multiprocessing\semaphore.c"
 				>
 			</File>
-			<File
-				RelativePath="..\Modules\_multiprocessing\win32_functions.c"
-				>
-			</File>
 		</Filter>
 	</Files>
 	<Globals>

PCbuild/pythoncore.vcproj

@@ -1054,6 +1054,10 @@
 				RelativePath="..\Modules\_weakref.c"
 				>
 			</File>
+			<File
+				RelativePath="..\Modules\_winapi.c"
+				>
+			</File>
 			<File
 				RelativePath="..\Modules\arraymodule.c"
 				>
@@ -1678,10 +1682,6 @@
 		<Filter
 			Name="PC"
 			>
-			<File
-				RelativePath="..\PC\_subprocess.c"
-				>
-			</File>
 			<File
 				RelativePath="..\PC\winreg.c"
 				>

setup.py

@@ -1387,7 +1387,6 @@ class PyBuildExt(build_ext):
         if platform == 'win32':
             multiprocessing_srcs = [ '_multiprocessing/multiprocessing.c',
                                      '_multiprocessing/semaphore.c',
-                                     '_multiprocessing/win32_functions.c'
                                    ]
 
         else: