Skip to content

C
cpython
Kaydet (Commit) 1aed624f authored tarafından Alexandre Vassalotti's avatar Alexandre Vassalotti
Dosyalara gözat
Seçenekler

Backport fast alternate io.BytesIO implementation. 

Merged r62778, r62779, r62802, r62806, r62807, r62808, r62809, r62844,
r62846, r62952, r62956.
üst 81673b7b
Expand all Show whitespace changes
Inline Side-by-side
Showing with 112 additions and 63 deletions
Lib/io.py
Dosyayı görüntüle @ 1aed624f
...@@ -494,6 +494,7 @@ class IOBase(object): ...@@ -494,6 +494,7 @@ class IOBase(object):
files, the newlines argument to open can be used to select the line files, the newlines argument to open can be used to select the line
terminator(s) recognized. terminator(s) recognized.
""" """
self._checkClosed()
if hasattr(self, "peek"): if hasattr(self, "peek"):
def nreadahead(): def nreadahead():
readahead = self.peek(1) readahead = self.peek(1)
...@@ -508,6 +509,8 @@ class IOBase(object): ...@@ -508,6 +509,8 @@ class IOBase(object):
return 1 return 1
if limit is None: if limit is None:
limit = -1 limit = -1
if not isinstance(limit, (int, long)):
raise TypeError("limit must be an integer")
res = bytearray() res = bytearray()
while limit < 0 or len(res) < limit: while limit < 0 or len(res) < limit:
b = self.read(nreadahead()) b = self.read(nreadahead())
...@@ -536,6 +539,10 @@ class IOBase(object): ...@@ -536,6 +539,10 @@ class IOBase(object):
lines so far exceeds hint. lines so far exceeds hint.
""" """
if hint is None: if hint is None:
hint = -1
if not isinstance(hint, (int, long)):
raise TypeError("hint must be an integer")
if hint <= 0:
return list(self) return list(self)
n = 0 n = 0
lines = [] lines = []
...@@ -686,7 +693,7 @@ class BufferedIOBase(IOBase): ...@@ -686,7 +693,7 @@ class BufferedIOBase(IOBase):
import array import array
if not isinstance(b, array.array): if not isinstance(b, array.array):
raise err raise err
b[:n] = array.array('b', data) b[:n] = array.array(b'b', data)
return n return n
def write(self, b): def write(self, b):
...@@ -729,6 +736,8 @@ class _BufferedIOMixin(BufferedIOBase): ...@@ -729,6 +736,8 @@ class _BufferedIOMixin(BufferedIOBase):
if pos is None: if pos is None:
pos = self.tell() pos = self.tell()
# XXX: Should seek() be used, instead of passing the position
# XXX directly to truncate?
return self.raw.truncate(pos) return self.raw.truncate(pos)
### Flush and close ### ### Flush and close ###
...@@ -768,7 +777,7 @@ class _BufferedIOMixin(BufferedIOBase): ...@@ -768,7 +777,7 @@ class _BufferedIOMixin(BufferedIOBase):
return self.raw.isatty() return self.raw.isatty()
class BytesIO(BufferedIOBase): class _BytesIO(BufferedIOBase):
"""Buffered I/O implementation using an in-memory bytes buffer.""" """Buffered I/O implementation using an in-memory bytes buffer."""
...@@ -777,20 +786,28 @@ class BytesIO(BufferedIOBase): ...@@ -777,20 +786,28 @@ class BytesIO(BufferedIOBase):
def __init__(self, initial_bytes=None): def __init__(self, initial_bytes=None):
buf = bytearray() buf = bytearray()
if initial_bytes is not None: if initial_bytes is not None:
buf += initial_bytes buf += bytearray(initial_bytes)
self._buffer = buf self._buffer = buf
self._pos = 0 self._pos = 0
def getvalue(self): def getvalue(self):
"""Return the bytes value (contents) of the buffer """Return the bytes value (contents) of the buffer
""" """
if self.closed:
raise ValueError("getvalue on closed file")
return bytes(self._buffer) return bytes(self._buffer)
def read(self, n=None): def read(self, n=None):
if self.closed:
raise ValueError("read from closed file")
if n is None: if n is None:
n = -1 n = -1
if not isinstance(n, (int, long)):
raise TypeError("argument must be an integer")
if n < 0: if n < 0:
n = len(self._buffer) n = len(self._buffer)
if len(self._buffer) <= self._pos:
return b""
newpos = min(len(self._buffer), self._pos + n) newpos = min(len(self._buffer), self._pos + n)
b = self._buffer[self._pos : newpos] b = self._buffer[self._pos : newpos]
self._pos = newpos self._pos = newpos
...@@ -807,6 +824,8 @@ class BytesIO(BufferedIOBase): ...@@ -807,6 +824,8 @@ class BytesIO(BufferedIOBase):
if isinstance(b, unicode): if isinstance(b, unicode):
raise TypeError("can't write unicode to binary stream") raise TypeError("can't write unicode to binary stream")
n = len(b) n = len(b)
if n == 0:
return 0
newpos = self._pos + n newpos = self._pos + n
if newpos > len(self._buffer): if newpos > len(self._buffer):
# Inserts null bytes between the current end of the file # Inserts null bytes between the current end of the file
...@@ -818,28 +837,38 @@ class BytesIO(BufferedIOBase): ...@@ -818,28 +837,38 @@ class BytesIO(BufferedIOBase):
return n return n
def seek(self, pos, whence=0): def seek(self, pos, whence=0):
if self.closed:
raise ValueError("seek on closed file")
try: try:
pos = pos.__index__() pos = pos.__index__()
except AttributeError as err: except AttributeError as err:
raise TypeError("an integer is required") # from err raise TypeError("an integer is required") # from err
if whence == 0: if whence == 0:
self._pos = max(0, pos) if pos < 0:
raise ValueError("negative seek position %r" % (pos,))
self._pos = pos
elif whence == 1: elif whence == 1:
self._pos = max(0, self._pos + pos) self._pos = max(0, self._pos + pos)
elif whence == 2: elif whence == 2:
self._pos = max(0, len(self._buffer) + pos) self._pos = max(0, len(self._buffer) + pos)
else: else:
raise IOError("invalid whence value") raise ValueError("invalid whence value")
return self._pos return self._pos
def tell(self): def tell(self):
if self.closed:
raise ValueError("tell on closed file")
return self._pos return self._pos
def truncate(self, pos=None): def truncate(self, pos=None):
if self.closed:
raise ValueError("truncate on closed file")
if pos is None: if pos is None:
pos = self._pos pos = self._pos
elif pos < 0:
raise ValueError("negative truncate position %r" % (pos,))
del self._buffer[pos:] del self._buffer[pos:]
return pos return self.seek(pos)
def readable(self): def readable(self):
return True return True
...@@ -850,6 +879,16 @@ class BytesIO(BufferedIOBase): ...@@ -850,6 +879,16 @@ class BytesIO(BufferedIOBase):
def seekable(self): def seekable(self):
return True return True
# Use the faster implementation of BytesIO if available
try:
import _bytesio
class BytesIO(_bytesio._BytesIO, BufferedIOBase):
__doc__ = _bytesio._BytesIO.__doc__
except ImportError:
BytesIO = _BytesIO
class BufferedReader(_BufferedIOMixin): class BufferedReader(_BufferedIOMixin):
...@@ -983,6 +1022,12 @@ class BufferedWriter(_BufferedIOMixin): ...@@ -983,6 +1022,12 @@ class BufferedWriter(_BufferedIOMixin):
raise BlockingIOError(e.errno, e.strerror, overage) raise BlockingIOError(e.errno, e.strerror, overage)
return written return written
def truncate(self, pos=None):
self.flush()
if pos is None:
pos = self.raw.tell()
return self.raw.truncate(pos)
def flush(self): def flush(self):
if self.closed: if self.closed:
raise ValueError("flush of closed file") raise ValueError("flush of closed file")
...@@ -1102,6 +1147,13 @@ class BufferedRandom(BufferedWriter, BufferedReader): ...@@ -1102,6 +1147,13 @@ class BufferedRandom(BufferedWriter, BufferedReader):
else: else:
return self.raw.tell() - len(self._read_buf) return self.raw.tell() - len(self._read_buf)
def truncate(self, pos=None):
if pos is None:
pos = self.tell()
# Use seek to flush the read buffer.
self.seek(pos)
return BufferedWriter.truncate(self)
def read(self, n=None): def read(self, n=None):
if n is None: if n is None:
n = -1 n = -1
...@@ -1150,11 +1202,7 @@ class TextIOBase(IOBase): ...@@ -1150,11 +1202,7 @@ class TextIOBase(IOBase):
def truncate(self, pos = None): def truncate(self, pos = None):
"""Truncate size to pos.""" """Truncate size to pos."""
self.flush() self._unsupported("truncate")
if pos is None:
pos = self.tell()
self.seek(pos)
return self.buffer.truncate()
def readline(self): def readline(self):
"""Read until newline or EOF. """Read until newline or EOF.
...@@ -1351,6 +1399,12 @@ class TextIOWrapper(TextIOBase): ...@@ -1351,6 +1399,12 @@ class TextIOWrapper(TextIOBase):
def seekable(self): def seekable(self):
return self._seekable return self._seekable
def readable(self):
return self.buffer.readable()
def writable(self):
return self.buffer.writable()
def flush(self): def flush(self):
self.buffer.flush() self.buffer.flush()
self._telling = self._seekable self._telling = self._seekable
...@@ -1542,7 +1596,16 @@ class TextIOWrapper(TextIOBase): ...@@ -1542,7 +1596,16 @@ class TextIOWrapper(TextIOBase):
finally: finally:
decoder.setstate(saved_state) decoder.setstate(saved_state)
def truncate(self, pos=None):
self.flush()
if pos is None:
pos = self.tell()
self.seek(pos)
return self.buffer.truncate()
def seek(self, cookie, whence=0): def seek(self, cookie, whence=0):
if self.closed:
raise ValueError("tell on closed file")
if not self._seekable: if not self._seekable:
raise IOError("underlying stream is not seekable") raise IOError("underlying stream is not seekable")
if whence == 1: # seek relative to current position if whence == 1: # seek relative to current position
...@@ -1629,8 +1692,12 @@ class TextIOWrapper(TextIOBase): ...@@ -1629,8 +1692,12 @@ class TextIOWrapper(TextIOBase):
return line return line
def readline(self, limit=None): def readline(self, limit=None):
if self.closed:
raise ValueError("read from closed file")
if limit is None: if limit is None:
limit = -1 limit = -1
if not isinstance(limit, (int, long)):
raise TypeError("limit must be an integer")
# Grab all the decoded text (we will rewind any extra bits later). # Grab all the decoded text (we will rewind any extra bits later).
line = self._get_decoded_chars() line = self._get_decoded_chars()
......
Lib/test/test_io.py
Dosyayı görüntüle @ 1aed624f
...@@ -97,7 +97,7 @@ class IOTest(unittest.TestCase): ...@@ -97,7 +97,7 @@ class IOTest(unittest.TestCase):
self.assertEqual(f.seek(-1, 2), 13) self.assertEqual(f.seek(-1, 2), 13)
self.assertEqual(f.tell(), 13) self.assertEqual(f.tell(), 13)
self.assertEqual(f.truncate(12), 12) self.assertEqual(f.truncate(12), 12)
self.assertEqual(f.tell(), 13) self.assertEqual(f.tell(), 12)
self.assertRaises(TypeError, f.seek, 0.0) self.assertRaises(TypeError, f.seek, 0.0)
def read_ops(self, f, buffered=False): def read_ops(self, f, buffered=False):
...@@ -142,7 +142,7 @@ class IOTest(unittest.TestCase): ...@@ -142,7 +142,7 @@ class IOTest(unittest.TestCase):
self.assertEqual(f.tell(), self.LARGE + 2) self.assertEqual(f.tell(), self.LARGE + 2)
self.assertEqual(f.seek(0, 2), self.LARGE + 2) self.assertEqual(f.seek(0, 2), self.LARGE + 2)
self.assertEqual(f.truncate(self.LARGE + 1), self.LARGE + 1) self.assertEqual(f.truncate(self.LARGE + 1), self.LARGE + 1)
self.assertEqual(f.tell(), self.LARGE + 2) self.assertEqual(f.tell(), self.LARGE + 1)
self.assertEqual(f.seek(0, 2), self.LARGE + 1) self.assertEqual(f.seek(0, 2), self.LARGE + 1)
self.assertEqual(f.seek(-1, 2), self.LARGE) self.assertEqual(f.seek(-1, 2), self.LARGE)
self.assertEqual(f.read(2), b"x") self.assertEqual(f.read(2), b"x")
...@@ -726,6 +726,7 @@ class TextIOWrapperTest(unittest.TestCase): ...@@ -726,6 +726,7 @@ class TextIOWrapperTest(unittest.TestCase):
txt.write("BB\nCCC\n") txt.write("BB\nCCC\n")
txt.write("X\rY\r\nZ") txt.write("X\rY\r\nZ")
txt.flush() txt.flush()
self.assertEquals(buf.closed, False)
self.assertEquals(buf.getvalue(), expected) self.assertEquals(buf.getvalue(), expected)
def testNewlines(self): def testNewlines(self):
...@@ -806,7 +807,8 @@ class TextIOWrapperTest(unittest.TestCase): ...@@ -806,7 +807,8 @@ class TextIOWrapperTest(unittest.TestCase):
txt = io.TextIOWrapper(buf, encoding="ascii", newline=newline) txt = io.TextIOWrapper(buf, encoding="ascii", newline=newline)
txt.write(data) txt.write(data)
txt.close() txt.close()
self.assertEquals(buf.getvalue(), expected) self.assertEquals(buf.closed, True)
self.assertRaises(ValueError, buf.getvalue)
finally: finally:
os.linesep = save_linesep os.linesep = save_linesep
......
Lib/test/test_memoryio.py 0 → 100644
Dosyayı görüntüle @ 1aed624f
This diff is collapsed.
Modules/_bytesio.c 0 → 100644
Dosyayı görüntüle @ 1aed624f
This diff is collapsed.
Modules/_fileio.c
Dosyayı görüntüle @ 1aed624f
...@@ -552,11 +552,10 @@ portable_lseek(int fd, PyObject *posobj, int whence) ...@@ -552,11 +552,10 @@ portable_lseek(int fd, PyObject *posobj, int whence)
PyErr_SetString(PyExc_TypeError, "an integer is required"); PyErr_SetString(PyExc_TypeError, "an integer is required");
return NULL; return NULL;
} }
#if !defined(HAVE_LARGEFILE_SUPPORT) #if defined(HAVE_LARGEFILE_SUPPORT)
pos = PyLong_AsLong(posobj); pos = PyLong_AsLongLong(posobj);
#else #else
pos = PyLong_Check(posobj) ? pos = PyLong_AsLong(posobj);
PyLong_AsLongLong(posobj) : PyLong_AsLong(posobj);
#endif #endif
if (PyErr_Occurred()) if (PyErr_Occurred())
return NULL; return NULL;
...@@ -572,10 +571,10 @@ portable_lseek(int fd, PyObject *posobj, int whence) ...@@ -572,10 +571,10 @@ portable_lseek(int fd, PyObject *posobj, int whence)
if (res < 0) if (res < 0)
return PyErr_SetFromErrno(PyExc_IOError); return PyErr_SetFromErrno(PyExc_IOError);
#if !defined(HAVE_LARGEFILE_SUPPORT) #if defined(HAVE_LARGEFILE_SUPPORT)
return PyLong_FromLong(res);
#else
return PyLong_FromLongLong(res); return PyLong_FromLongLong(res);
#else
return PyLong_FromLong(res);
#endif #endif
} }
...@@ -622,48 +621,29 @@ fileio_truncate(PyFileIOObject *self, PyObject *args) ...@@ -622,48 +621,29 @@ fileio_truncate(PyFileIOObject *self, PyObject *args)
return NULL; return NULL;
if (posobj == Py_None || posobj == NULL) { if (posobj == Py_None || posobj == NULL) {
/* Get the current position. */
posobj = portable_lseek(fd, NULL, 1); posobj = portable_lseek(fd, NULL, 1);
if (posobj == NULL) if (posobj == NULL)
return NULL; return NULL;
} }
else { else {
Py_INCREF(posobj); /* Move to the position to be truncated. */
posobj = portable_lseek(fd, posobj, 0);
} }
#if !defined(HAVE_LARGEFILE_SUPPORT) #if defined(HAVE_LARGEFILE_SUPPORT)
pos = PyLong_AsLong(posobj); pos = PyLong_AsLongLong(posobj);
#else #else
pos = PyLong_Check(posobj) ? pos = PyLong_AsLong(posobj);
PyLong_AsLongLong(posobj) : PyLong_AsLong(posobj);
#endif #endif
if (PyErr_Occurred()) { if (PyErr_Occurred())
Py_DECREF(posobj);
return NULL; return NULL;
}
#ifdef MS_WINDOWS #ifdef MS_WINDOWS
/* MS _chsize doesn't work if newsize doesn't fit in 32 bits, /* MS _chsize doesn't work if newsize doesn't fit in 32 bits,
so don't even try using it. */ so don't even try using it. */
{ {
HANDLE hFile; HANDLE hFile;
PyObject *pos2, *oldposobj;
/* store the current position */
oldposobj = portable_lseek(self->fd, NULL, 1);
if (oldposobj == NULL) {
Py_DECREF(posobj);
return NULL;
}
/* Have to move current pos to desired endpoint on Windows. */
errno = 0;
pos2 = portable_lseek(fd, posobj, SEEK_SET);
if (pos2 == NULL) {
Py_DECREF(posobj);
Py_DECREF(oldposobj);
return NULL;
}
Py_DECREF(pos2);
/* Truncate. Note that this may grow the file! */ /* Truncate. Note that this may grow the file! */
Py_BEGIN_ALLOW_THREADS Py_BEGIN_ALLOW_THREADS
...@@ -676,18 +656,6 @@ fileio_truncate(PyFileIOObject *self, PyObject *args) ...@@ -676,18 +656,6 @@ fileio_truncate(PyFileIOObject *self, PyObject *args)
errno = EACCES; errno = EACCES;
} }
Py_END_ALLOW_THREADS Py_END_ALLOW_THREADS
if (ret == 0) {
/* Move to the previous position in the file */
pos2 = portable_lseek(fd, oldposobj, SEEK_SET);
if (pos2 == NULL) {
Py_DECREF(posobj);
Py_DECREF(oldposobj);
return NULL;
}
}
Py_DECREF(pos2);
Py_DECREF(oldposobj);
} }
#else #else
Py_BEGIN_ALLOW_THREADS Py_BEGIN_ALLOW_THREADS
...@@ -697,7 +665,6 @@ fileio_truncate(PyFileIOObject *self, PyObject *args) ...@@ -697,7 +665,6 @@ fileio_truncate(PyFileIOObject *self, PyObject *args)
#endif /* !MS_WINDOWS */ #endif /* !MS_WINDOWS */
if (ret != 0) { if (ret != 0) {
Py_DECREF(posobj);
PyErr_SetFromErrno(PyExc_IOError); PyErr_SetFromErrno(PyExc_IOError);
return NULL; return NULL;
} }
...@@ -791,7 +758,8 @@ PyDoc_STRVAR(seek_doc, ...@@ -791,7 +758,8 @@ PyDoc_STRVAR(seek_doc,
PyDoc_STRVAR(truncate_doc, PyDoc_STRVAR(truncate_doc,
"truncate([size: int]) -> None. Truncate the file to at most size bytes.\n" "truncate([size: int]) -> None. Truncate the file to at most size bytes.\n"
"\n" "\n"
"Size defaults to the current file position, as returned by tell()."); "Size defaults to the current file position, as returned by tell()."
"The current file position is changed to the value of size.");
#endif #endif
PyDoc_STRVAR(tell_doc, PyDoc_STRVAR(tell_doc,
......
Objects/bytesobject.c
Dosyayı görüntüle @ 1aed624f
...@@ -315,8 +315,8 @@ bytes_iconcat(PyBytesObject *self, PyObject *other) ...@@ -315,8 +315,8 @@ bytes_iconcat(PyBytesObject *self, PyObject *other)
Py_buffer vo; Py_buffer vo;
if (_getbuffer(other, &vo) < 0) { if (_getbuffer(other, &vo) < 0) {
PyErr_Format(PyExc_TypeError, "can't concat bytes to %.100s", PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s",
Py_TYPE(self)->tp_name); Py_TYPE(other)->tp_name, Py_TYPE(self)->tp_name);
return NULL; return NULL;
} }
......
PC/VS8.0/pythoncore.vcproj
Dosyayı görüntüle @ 1aed624f
...@@ -990,6 +990,10 @@ ...@@ -990,6 +990,10 @@
RelativePath="..\..\Modules\_fileio.c" RelativePath="..\..\Modules\_fileio.c"
> >
</File> </File>
<File
RelativePath="..\..\Modules\_bytesio.c"
>
</File>
<File <File
RelativePath="..\..\Modules\_functoolsmodule.c" RelativePath="..\..\Modules\_functoolsmodule.c"
> >
......
PC/config.c
Dosyayı görüntüle @ 1aed624f
...@@ -53,6 +53,7 @@ extern void init_winreg(void); ...@@ -53,6 +53,7 @@ extern void init_winreg(void);
extern void init_struct(void); extern void init_struct(void);
extern void initdatetime(void); extern void initdatetime(void);
extern void init_fileio(void); extern void init_fileio(void);
extern void init_bytesio(void);
extern void init_functools(void); extern void init_functools(void);
extern void init_json(void); extern void init_json(void);
extern void initzlib(void); extern void initzlib(void);
...@@ -132,6 +133,7 @@ struct _inittab _PyImport_Inittab[] = { ...@@ -132,6 +133,7 @@ struct _inittab _PyImport_Inittab[] = {
{"_struct", init_struct}, {"_struct", init_struct},
{"datetime", initdatetime}, {"datetime", initdatetime},
{"_fileio", init_fileio}, {"_fileio", init_fileio},
{"_bytesio", init_bytesio},
{"_functools", init_functools}, {"_functools", init_functools},
{"_json", init_json}, {"_json", init_json},
......
PCbuild/pythoncore.vcproj
Dosyayı görüntüle @ 1aed624f
...@@ -994,6 +994,10 @@ ...@@ -994,6 +994,10 @@
RelativePath="..\Modules\_fileio.c" RelativePath="..\Modules\_fileio.c"
> >
</File> </File>
<File
RelativePath="..\Modules\_bytesio.c"
>
</File>
<File <File
RelativePath="..\Modules\_functoolsmodule.c" RelativePath="..\Modules\_functoolsmodule.c"
> >
......
setup.py
Dosyayı görüntüle @ 1aed624f
...@@ -436,6 +436,8 @@ class PyBuildExt(build_ext): ...@@ -436,6 +436,8 @@ class PyBuildExt(build_ext):
exts.append( Extension('operator', ['operator.c']) ) exts.append( Extension('operator', ['operator.c']) )
# Python 3.0 _fileio module # Python 3.0 _fileio module
exts.append( Extension("_fileio", ["_fileio.c"]) ) exts.append( Extension("_fileio", ["_fileio.c"]) )
# Python 3.0 _bytesio module
exts.append( Extension("_bytesio", ["_bytesio.c"]) )
# _functools # _functools
exts.append( Extension("_functools", ["_functoolsmodule.c"]) ) exts.append( Extension("_functools", ["_functoolsmodule.c"]) )
# _json speedups # _json speedups
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment