Skip to content

C
cpython
Kaydet (Commit) f69d94f6 authored tarafından Andrew McNamara's avatar Andrew McNamara
Dosyalara gözat
Seçenekler

Moved reader \r and \n processing from the iterator to the state machine - 

this allows for better handling of newline characters in quoted fields (and
hopefully resolves Bug 967934).
üst a1974c14
Hide whitespace changes
Inline Side-by-side
Showing with 78 additions and 99 deletions
Misc/NEWS
Dosyayı görüntüle @ f69d94f6
......@@ -48,10 +48,11 @@ Library
dictates.
+ the parser now removes the escapechar prefix from escaped characters.
+ when quoting=QUOTE_NONNUMERIC, the writer now tests for numeric
objects, rather than attempting to cast to float, and using the
success of that as the determinator.
types, rather than any object than can be represented as a numeric.
+ when quoting=QUOTE_NONNUMERIC, the reader now casts unquoted fields
to floats.
+ reader now allows \r characters to be quoted (previously it only allowed
\n to be quoted).
+ writer doublequote handling improved.
+ Dialect classes passed to the module are no longer instantiated by
the module before being parsed (the former validation scheme required
......
Modules/_csv.c
Dosyayı görüntüle @ f69d94f6
......@@ -48,7 +48,8 @@ static long field_limit = 128 * 1024; /* max parsed field size */
typedef enum {
START_RECORD, START_FIELD, ESCAPED_CHAR, IN_FIELD,
IN_QUOTED_FIELD, ESCAPE_IN_QUOTED_FIELD, QUOTE_IN_QUOTED_FIELD
IN_QUOTED_FIELD, ESCAPE_IN_QUOTED_FIELD, QUOTE_IN_QUOTED_FIELD,
EAT_CRNL
} ParserState;
typedef enum {
......@@ -96,7 +97,6 @@ typedef struct {
char *field; /* build current field in here */
int field_size; /* size of allocated buffer */
int field_len; /* length of current field */
int had_parse_error; /* did we have a parse error? */
int numeric_field; /* treat field as numeric */
unsigned long line_num; /* Source-file line number */
} ReaderObj;
......@@ -497,6 +497,9 @@ _call_dialect(PyObject *dialect_inst, PyObject *kwargs)
return dialect;
}
/*
* READER
*/
static int
parse_save_field(ReaderObj *self)
{
......@@ -543,22 +546,6 @@ parse_grow_buff(ReaderObj *self)
return 1;
}
static int
parse_reset(ReaderObj *self)
{
if (self->fields) {
Py_DECREF(self->fields);
}
self->fields = PyList_New(0);
if (self->fields == NULL)
return -1;
self->field_len = 0;
self->state = START_RECORD;
self->had_parse_error = 0;
self->numeric_field = 0;
return 0;
}
static int
parse_add_char(ReaderObj *self, char c)
{
......@@ -581,19 +568,23 @@ parse_process_char(ReaderObj *self, char c)
switch (self->state) {
case START_RECORD:
/* start of record */
if (c == '\n')
if (c == '\0')
/* empty line - return [] */
break;
else if (c == '\n' || c == '\r') {
self->state = EAT_CRNL;
break;
}
/* normal character - handle as START_FIELD */
self->state = START_FIELD;
/* fallthru */
case START_FIELD:
/* expecting field */
if (c == '\n') {
if (c == '\n' || c == '\r' || c == '\0') {
/* save empty field - return [fields] */
if (parse_save_field(self) < 0)
return -1;
self->state = START_RECORD;
self->state = (c == '\0' ? START_RECORD : EAT_CRNL);
}
else if (c == dialect->quotechar &&
dialect->quoting != QUOTE_NONE) {
......@@ -623,6 +614,8 @@ parse_process_char(ReaderObj *self, char c)
break;
case ESCAPED_CHAR:
if (c == '\0')
c = '\n';
if (parse_add_char(self, c) < 0)
return -1;
self->state = IN_FIELD;
......@@ -630,11 +623,11 @@ parse_process_char(ReaderObj *self, char c)
case IN_FIELD:
/* in unquoted field */
if (c == '\n') {
if (c == '\n' || c == '\r' || c == '\0') {
/* end of line - return [fields] */
if (parse_save_field(self) < 0)
return -1;
self->state = START_RECORD;
self->state = (c == '\0' ? START_RECORD : EAT_CRNL);
}
else if (c == dialect->escapechar) {
/* possible escaped character */
......@@ -655,11 +648,8 @@ parse_process_char(ReaderObj *self, char c)
case IN_QUOTED_FIELD:
/* in quoted field */
if (c == '\n') {
/* end of line - save '\n' in field */
if (parse_add_char(self, '\n') < 0)
return -1;
}
if (c == '\0')
;
else if (c == dialect->escapechar) {
/* Possible escape character */
self->state = ESCAPE_IN_QUOTED_FIELD;
......@@ -683,6 +673,8 @@ parse_process_char(ReaderObj *self, char c)
break;
case ESCAPE_IN_QUOTED_FIELD:
if (c == '\0')
c = '\n';
if (parse_add_char(self, c) < 0)
return -1;
self->state = IN_QUOTED_FIELD;
......@@ -703,11 +695,11 @@ parse_process_char(ReaderObj *self, char c)
return -1;
self->state = START_FIELD;
}
else if (c == '\n') {
else if (c == '\n' || c == '\r' || c == '\0') {
/* end of line - return [fields] */
if (parse_save_field(self) < 0)
return -1;
self->state = START_RECORD;
self->state = (c == '\0' ? START_RECORD : EAT_CRNL);
}
else if (!dialect->strict) {
if (parse_add_char(self, c) < 0)
......@@ -716,7 +708,6 @@ parse_process_char(ReaderObj *self, char c)
}
else {
/* illegal */
self->had_parse_error = 1;
PyErr_Format(error_obj, "'%c' expected after '%c'",
dialect->delimiter,
dialect->quotechar);
......@@ -724,104 +715,83 @@ parse_process_char(ReaderObj *self, char c)
}
break;
case EAT_CRNL:
if (c == '\n' || c == '\r')
;
else if (c == '\0')
self->state = START_RECORD;
else {
PyErr_Format(error_obj, "new-line character seen in unquoted field - do you need to open the file in universal-newline mode?");
return -1;
}
break;
}
return 0;
}
/*
* READER
*/
#define R_OFF(x) offsetof(ReaderObj, x)
static struct PyMemberDef Reader_memberlist[] = {
{ "dialect", T_OBJECT, R_OFF(dialect), RO },
{ "line_num", T_ULONG, R_OFF(line_num), RO },
{ NULL }
};
static int
parse_reset(ReaderObj *self)
{
Py_XDECREF(self->fields);
self->fields = PyList_New(0);
if (self->fields == NULL)
return -1;
self->field_len = 0;
self->state = START_RECORD;
self->numeric_field = 0;
return 0;
}
static PyObject *
Reader_iternext(ReaderObj *self)
{
PyObject *lineobj;
PyObject *fields;
char *line;
PyObject *fields = NULL;
char *line, c;
int linelen;
if (parse_reset(self) < 0)
return NULL;
do {
lineobj = PyIter_Next(self->input_iter);
if (lineobj == NULL) {
/* End of input OR exception */
if (!PyErr_Occurred() && self->field_len != 0)
return PyErr_Format(error_obj,
"newline inside string");
PyErr_Format(error_obj,
"newline inside string");
return NULL;
}
++self->line_num;
if (self->had_parse_error)
if (parse_reset(self) < 0) {
Py_DECREF(lineobj);
return NULL;
}
line = PyString_AsString(lineobj);
linelen = PyString_Size(lineobj);
if (line == NULL) {
if (line == NULL || linelen < 0) {
Py_DECREF(lineobj);
return NULL;
}
if (strlen(line) < (size_t)PyString_GET_SIZE(lineobj)) {
self->had_parse_error = 1;
Py_DECREF(lineobj);
return PyErr_Format(error_obj,
"string with NUL bytes");
}
/* Process line of text - send '\n' to processing code to
represent end of line. End of line which is not at end of
string is an error. */
while (*line) {
char c;
c = *line++;
if (c == '\r') {
c = *line++;
if (c == '\0')
/* macintosh end of line */
break;
if (c == '\n') {
c = *line++;
if (c == '\0')
/* DOS end of line */
break;
}
self->had_parse_error = 1;
Py_DECREF(lineobj);
return PyErr_Format(error_obj,
"newline inside string");
}
if (c == '\n') {
c = *line++;
if (c == '\0')
/* unix end of line */
break;
self->had_parse_error = 1;
Py_DECREF(lineobj);
return PyErr_Format(error_obj,
"newline inside string");
}
while (linelen--) {
c = *line++;
if (c == '\0') {
Py_DECREF(lineobj);
PyErr_Format(error_obj,
"line contains NULL byte");
goto err;
}
if (parse_process_char(self, c) < 0) {
Py_DECREF(lineobj);
return NULL;
goto err;
}
}
if (parse_process_char(self, '\n') < 0) {
Py_DECREF(lineobj);
return NULL;
}
Py_DECREF(lineobj);
if (parse_process_char(self, 0) < 0)
goto err;
} while (self->state != START_RECORD);
fields = self->fields;
self->fields = PyList_New(0);
self->fields = NULL;
err:
return fields;
}
......@@ -875,6 +845,14 @@ PyDoc_STRVAR(Reader_Type_doc,
static struct PyMethodDef Reader_methods[] = {
{ NULL, NULL }
};
#define R_OFF(x) offsetof(ReaderObj, x)
static struct PyMemberDef Reader_memberlist[] = {
{ "dialect", T_OBJECT, R_OFF(dialect), RO },
{ "line_num", T_ULONG, R_OFF(line_num), RO },
{ NULL }
};
static PyTypeObject Reader_Type = {
PyObject_HEAD_INIT(NULL)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment