Kaydet (Commit) d82c3105 authored tarafından Bob Ippolito's avatar Bob Ippolito

Apply revised patch for GzipFile.readline performance #1281707

üst 763b50f9
...@@ -107,6 +107,8 @@ class GzipFile: ...@@ -107,6 +107,8 @@ class GzipFile:
self.extrabuf = "" self.extrabuf = ""
self.extrasize = 0 self.extrasize = 0
self.filename = filename self.filename = filename
# Starts small, scales exponentially
self.min_readsize = 100
elif mode[0:1] == 'w' or mode[0:1] == 'a': elif mode[0:1] == 'w' or mode[0:1] == 'a':
self.mode = WRITE self.mode = WRITE
...@@ -381,32 +383,35 @@ class GzipFile: ...@@ -381,32 +383,35 @@ class GzipFile:
self.read(count % 1024) self.read(count % 1024)
def readline(self, size=-1): def readline(self, size=-1):
if size < 0: size = sys.maxint if size < 0:
size = sys.maxint
readsize = self.min_readsize
else:
readsize = size
bufs = [] bufs = []
readsize = min(100, size) # Read from the file in small chunks while size != 0:
while True:
if size == 0:
return "".join(bufs) # Return resulting line
c = self.read(readsize) c = self.read(readsize)
i = c.find('\n') i = c.find('\n')
if size is not None:
# We set i=size to break out of the loop under two # We set i=size to break out of the loop under two
# conditions: 1) there's no newline, and the chunk is # conditions: 1) there's no newline, and the chunk is
# larger than size, or 2) there is a newline, but the # larger than size, or 2) there is a newline, but the
# resulting line would be longer than 'size'. # resulting line would be longer than 'size'.
if i==-1 and len(c) > size: i=size-1 if (size <= i) or (i == -1 and len(c) > size):
elif size <= i: i = size -1 i = size - 1
if i >= 0 or c == '': if i >= 0 or c == '':
bufs.append(c[:i+1]) # Add portion of last chunk bufs.append(c[:i + 1]) # Add portion of last chunk
self._unread(c[i+1:]) # Push back rest of chunk self._unread(c[i + 1:]) # Push back rest of chunk
return ''.join(bufs) # Return resulting line break
# Append chunk to list, decrease 'size', # Append chunk to list, decrease 'size',
bufs.append(c) bufs.append(c)
size = size - len(c) size = size - len(c)
readsize = min(size, readsize * 2) readsize = min(size, readsize * 2)
if readsize > self.min_readsize:
self.min_readsize = min(readsize, self.min_readsize * 2, 512)
return ''.join(bufs) # Return resulting line
def readlines(self, sizehint=0): def readlines(self, sizehint=0):
# Negative numbers result in reading all the lines # Negative numbers result in reading all the lines
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment