Skip to content

C
cpython
Kaydet (Commit) 3a4586a9 authored tarafından Charles-François Natali's avatar Charles-François Natali
Dosyalara gözat
Seçenekler

Issue #18923: Update subprocess to use the new selectors module.

üst 2ce6c44a
Show whitespace changes
Inline Side-by-side
Showing with 71 additions and 169 deletions
Lib/subprocess.py
Dosyayı görüntüle @ 3a4586a9
...@@ -404,15 +404,23 @@ if mswindows: ...@@ -404,15 +404,23 @@ if mswindows:
hStdError = None hStdError = None
wShowWindow = 0 wShowWindow = 0
else: else:
import select
_has_poll = hasattr(select, 'poll')
import _posixsubprocess import _posixsubprocess
import select
import selectors
# When select or poll has indicated that the file is writable, # When select or poll has indicated that the file is writable,
# we can write up to _PIPE_BUF bytes without risk of blocking. # we can write up to _PIPE_BUF bytes without risk of blocking.
# POSIX defines PIPE_BUF as >= 512. # POSIX defines PIPE_BUF as >= 512.
_PIPE_BUF = getattr(select, 'PIPE_BUF', 512) _PIPE_BUF = getattr(select, 'PIPE_BUF', 512)
# poll/select have the advantage of not requiring any extra file
# descriptor, contrarily to epoll/kqueue (also, they require a single
# syscall).
if hasattr(selectors, 'PollSelector'):
_PopenSelector = selectors.PollSelector
else:
_PopenSelector = selectors.SelectSelector
__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput", __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput",
"getoutput", "check_output", "CalledProcessError", "DEVNULL"] "getoutput", "check_output", "CalledProcessError", "DEVNULL"]
...@@ -1530,206 +1538,98 @@ class Popen(object): ...@@ -1530,206 +1538,98 @@ class Popen(object):
if not input: if not input:
self.stdin.close() self.stdin.close()
if _has_poll: stdout = None
stdout, stderr = self._communicate_with_poll(input, endtime, stderr = None
orig_timeout)
else:
stdout, stderr = self._communicate_with_select(input, endtime,
orig_timeout)
self.wait(timeout=self._remaining_time(endtime))
# All data exchanged. Translate lists into strings.
if stdout is not None:
stdout = b''.join(stdout)
if stderr is not None:
stderr = b''.join(stderr)
# Translate newlines, if requested.
# This also turns bytes into strings.
if self.universal_newlines:
if stdout is not None:
stdout = self._translate_newlines(stdout,
self.stdout.encoding)
if stderr is not None:
stderr = self._translate_newlines(stderr,
self.stderr.encoding)
return (stdout, stderr)
def _save_input(self, input):
# This method is called from the _communicate_with_*() methods
# so that if we time out while communicating, we can continue
# sending input if we retry.
if self.stdin and self._input is None:
self._input_offset = 0
self._input = input
if self.universal_newlines and input is not None:
self._input = self._input.encode(self.stdin.encoding)
def _communicate_with_poll(self, input, endtime, orig_timeout):
stdout = None # Return
stderr = None # Return
if not self._communication_started:
self._fd2file = {}
poller = select.poll()
def register_and_append(file_obj, eventmask):
poller.register(file_obj.fileno(), eventmask)
self._fd2file[file_obj.fileno()] = file_obj
def close_unregister_and_remove(fd):
poller.unregister(fd)
self._fd2file[fd].close()
self._fd2file.pop(fd)
if self.stdin and input:
register_and_append(self.stdin, select.POLLOUT)
# Only create this mapping if we haven't already. # Only create this mapping if we haven't already.
if not self._communication_started: if not self._communication_started:
self._fd2output = {} self._fileobj2output = {}
if self.stdout: if self.stdout:
self._fd2output[self.stdout.fileno()] = [] self._fileobj2output[self.stdout] = []
if self.stderr: if self.stderr:
self._fd2output[self.stderr.fileno()] = [] self._fileobj2output[self.stderr] = []
select_POLLIN_POLLPRI = select.POLLIN | select.POLLPRI
if self.stdout: if self.stdout:
register_and_append(self.stdout, select_POLLIN_POLLPRI) stdout = self._fileobj2output[self.stdout]
stdout = self._fd2output[self.stdout.fileno()]
if self.stderr: if self.stderr:
register_and_append(self.stderr, select_POLLIN_POLLPRI) stderr = self._fileobj2output[self.stderr]
stderr = self._fd2output[self.stderr.fileno()]
self._save_input(input) self._save_input(input)
while self._fd2file: with _PopenSelector() as selector:
timeout = self._remaining_time(endtime)
if timeout is not None and timeout < 0:
raise TimeoutExpired(self.args, orig_timeout)
try:
ready = poller.poll(timeout)
except OSError as e:
if e.args[0] == errno.EINTR:
continue
raise
self._check_timeout(endtime, orig_timeout)
# XXX Rewrite these to use non-blocking I/O on the
# file objects; they are no longer using C stdio!
for fd, mode in ready:
if mode & select.POLLOUT:
chunk = self._input[self._input_offset :
self._input_offset + _PIPE_BUF]
try:
self._input_offset += os.write(fd, chunk)
except OSError as e:
if e.errno == errno.EPIPE:
close_unregister_and_remove(fd)
else:
raise
else:
if self._input_offset >= len(self._input):
close_unregister_and_remove(fd)
elif mode & select_POLLIN_POLLPRI:
data = os.read(fd, 4096)
if not data:
close_unregister_and_remove(fd)
self._fd2output[fd].append(data)
else:
# Ignore hang up or errors.
close_unregister_and_remove(fd)
return (stdout, stderr)
def _communicate_with_select(self, input, endtime, orig_timeout):
if not self._communication_started:
self._read_set = []
self._write_set = []
if self.stdin and input: if self.stdin and input:
self._write_set.append(self.stdin) selector.register(self.stdin, selectors.EVENT_WRITE)
if self.stdout: if self.stdout:
self._read_set.append(self.stdout) selector.register(self.stdout, selectors.EVENT_READ)
if self.stderr: if self.stderr:
self._read_set.append(self.stderr) selector.register(self.stderr, selectors.EVENT_READ)
self._save_input(input)
stdout = None # Return while selector.get_map():
stderr = None # Return
if self.stdout:
if not self._communication_started:
self._stdout_buff = []
stdout = self._stdout_buff
if self.stderr:
if not self._communication_started:
self._stderr_buff = []
stderr = self._stderr_buff
while self._read_set or self._write_set:
timeout = self._remaining_time(endtime) timeout = self._remaining_time(endtime)
if timeout is not None and timeout < 0: if timeout is not None and timeout < 0:
raise TimeoutExpired(self.args, orig_timeout) raise TimeoutExpired(self.args, orig_timeout)
try:
(rlist, wlist, xlist) = \
select.select(self._read_set, self._write_set, [],
timeout)
except OSError as e:
if e.args[0] == errno.EINTR:
continue
raise
# According to the docs, returning three empty lists indicates ready = selector.select(timeout)
# that the timeout expired.
if not (rlist or wlist or xlist):
raise TimeoutExpired(self.args, orig_timeout)
# We also check what time it is ourselves for good measure.
self._check_timeout(endtime, orig_timeout) self._check_timeout(endtime, orig_timeout)
# XXX Rewrite these to use non-blocking I/O on the # XXX Rewrite these to use non-blocking I/O on the file
# file objects; they are no longer using C stdio! # objects; they are no longer using C stdio!
if self.stdin in wlist: for key, events in ready:
if key.fileobj is self.stdin:
chunk = self._input[self._input_offset : chunk = self._input[self._input_offset :
self._input_offset + _PIPE_BUF] self._input_offset + _PIPE_BUF]
try: try:
bytes_written = os.write(self.stdin.fileno(), chunk) self._input_offset += os.write(key.fd, chunk)
except OSError as e: except OSError as e:
if e.errno == errno.EPIPE: if e.errno == errno.EPIPE:
self.stdin.close() selector.unregister(key.fileobj)
self._write_set.remove(self.stdin) key.fileobj.close()
else: else:
raise raise
else: else:
self._input_offset += bytes_written
if self._input_offset >= len(self._input): if self._input_offset >= len(self._input):
self.stdin.close() selector.unregister(key.fileobj)
self._write_set.remove(self.stdin) key.fileobj.close()
elif key.fileobj in (self.stdout, self.stderr):
if self.stdout in rlist: data = os.read(key.fd, 4096)
data = os.read(self.stdout.fileno(), 1024)
if not data: if not data:
self.stdout.close() selector.unregister(key.fileobj)
self._read_set.remove(self.stdout) key.fileobj.close()
stdout.append(data) self._fileobj2output[key.fileobj].append(data)
if self.stderr in rlist: self.wait(timeout=self._remaining_time(endtime))
data = os.read(self.stderr.fileno(), 1024)
if not data: # All data exchanged. Translate lists into strings.
self.stderr.close() if stdout is not None:
self._read_set.remove(self.stderr) stdout = b''.join(stdout)
stderr.append(data) if stderr is not None:
stderr = b''.join(stderr)
# Translate newlines, if requested.
# This also turns bytes into strings.
if self.universal_newlines:
if stdout is not None:
stdout = self._translate_newlines(stdout,
self.stdout.encoding)
if stderr is not None:
stderr = self._translate_newlines(stderr,
self.stderr.encoding)
return (stdout, stderr) return (stdout, stderr)
def _save_input(self, input):
# This method is called from the _communicate_with_*() methods
# so that if we time out while communicating, we can continue
# sending input if we retry.
if self.stdin and self._input is None:
self._input_offset = 0
self._input = input
if self.universal_newlines and input is not None:
self._input = self._input.encode(self.stdin.encoding)
def send_signal(self, sig): def send_signal(self, sig):
"""Send a signal to the process """Send a signal to the process
""" """
......
Lib/test/test_subprocess.py
Dosyayı görüntüle @ 3a4586a9
...@@ -11,6 +11,7 @@ import errno ...@@ -11,6 +11,7 @@ import errno
import tempfile import tempfile
import time import time
import re import re
import selectors
import sysconfig import sysconfig
import warnings import warnings
import select import select
...@@ -2179,15 +2180,16 @@ class CommandTests(unittest.TestCase): ...@@ -2179,15 +2180,16 @@ class CommandTests(unittest.TestCase):
os.rmdir(dir) os.rmdir(dir)
@unittest.skipUnless(getattr(subprocess, '_has_poll', False), @unittest.skipUnless(hasattr(selectors, 'PollSelector'),
"poll system call not supported") "Test needs selectors.PollSelector")
class ProcessTestCaseNoPoll(ProcessTestCase): class ProcessTestCaseNoPoll(ProcessTestCase):
def setUp(self): def setUp(self):
subprocess._has_poll = False self.orig_selector = subprocess._PopenSelector
subprocess._PopenSelector = selectors.SelectSelector
ProcessTestCase.setUp(self) ProcessTestCase.setUp(self)
def tearDown(self): def tearDown(self):
subprocess._has_poll = True subprocess._PopenSelector = self.orig_selector
ProcessTestCase.tearDown(self) ProcessTestCase.tearDown(self)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment