Mass checkin of universal newline support.

Highlights: import and friends will understand any of \r, \n and \r\n
as end of line. Python file input will do the same if you use mode 'U'.
Everything can be disabled by configuring with --without-universal-newlines.

See PEP278 for details.

Include/fileobject.h

@@ -14,8 +14,13 @@ typedef struct {
 	PyObject *f_mode;
 	int (*f_close)(FILE *);
 	int f_softspace; /* Flag used by 'print' command */
-	int f_binary; /* Flag which indicates whether the file is
+	int f_binary; /* Flag which indicates whether the file is open
 			 open in binary (1) or test (0) mode */
+#ifdef WITH_UNIVERSAL_NEWLINES
+	int f_univ_newline;	/* Handle any newline convention */
+	int f_newlinetypes;	/* Types of newlines seen */
+	int f_skipnextlf;	/* Skip next \n */
+#endif
 } PyFileObject;
 
 extern DL_IMPORT(PyTypeObject) PyFile_Type;
@@ -40,6 +45,19 @@ extern DL_IMPORT(int) PyObject_AsFileDescriptor(PyObject *);
 */
 extern DL_IMPORT(const char *) Py_FileSystemDefaultEncoding;
 
+#ifdef WITH_UNIVERSAL_NEWLINES
+/* Routines to replace fread() and fgets() which accept any of \r, \n
+   or \r\n as line terminators.
+*/
+#define PY_STDIOTEXTMODE "b"
+char *Py_UniversalNewlineFgets(char *, int, FILE*, PyObject *);
+size_t Py_UniversalNewlineFread(void *, size_t, FILE *, PyObject *);
+#else
+#define PY_STDIOTEXTMODE ""
+#define Py_UniversalNewlineFgets(buf, len, fp, obj) (fgets((buf), (len), (fp)))
+#define Py_UniversalNewlineFread(buf, len, fp, obj) \
+		(fread((buf), 1, (len), (fp)))
+#endif /* WITH_UNIVERSAL_NEWLINES */
 #ifdef __cplusplus
 }
 #endif

Lib/linecache.py

@@ -90,7 +90,7 @@ def updatecache(filename):
 ##          print '*** Cannot stat', filename, ':', msg
             return []
     try:
-        fp = open(fullname, 'r')
+        fp = open(fullname, 'rU')
         lines = fp.readlines()
         fp.close()
     except IOError, msg:

Lib/py_compile.py

@@ -44,7 +44,7 @@ def compile(file, cfile=None, dfile=None):
 
     """
     import os, marshal, __builtin__
-    f = open(file)
+    f = open(file, 'U')
     try:
         timestamp = long(os.fstat(f.fileno())[8])
     except AttributeError:

Mac/Python/macmain.c

@@ -611,7 +611,7 @@ Py_Main(int argc, char **argv, char *filename)
 			Py_GetVersion(), Py_GetPlatform(), COPYRIGHT);
 	
 	if (filename != NULL) {
-		if ((fp = fopen(filename, "r")) == NULL) {
+		if ((fp = fopen(filename, "r" PY_STDIOTEXTMODE)) == NULL) {
 			fprintf(stderr, "%s: can't open file '%s'\n",
 				argv[0], filename);
 			PyMac_Exit(2);
@@ -630,7 +630,7 @@ Py_Main(int argc, char **argv, char *filename)
 	PySys_SetArgv(argc, argv);
 
 	if (filename == NULL && isatty((int)fileno(fp))) {
-		FILE *fp = fopen(STARTUP, "r");
+		FILE *fp = fopen(STARTUP, "r" PY_STDIOTEXTMODE);
 		if (fp != NULL) {
 			(void) PyRun_SimpleFile(fp, STARTUP);
 			PyErr_Clear();

Parser/pgenmain.c

@@ -13,6 +13,7 @@
    - check for duplicate definitions of names (instead of fatal err)
 */
 
+#include "Python.h"
 #include "pgenheaders.h"
 #include "grammar.h"
 #include "node.h"
@@ -182,6 +183,16 @@ PyOS_Readline(char *prompt)
 	return PyMem_REALLOC(p, n+1);
 }
 
+#ifdef WITH_UNIVERSAL_NEWLINES
+/* No-nonsense fgets */
+char *
+Py_UniversalNewlineFgets(char *buf, int n, FILE *stream, PyObject *fobj)
+{
+	return fgets(buf, n, stream);
+}
+#endif
+
+
 #include <stdarg.h>
 
 void

Parser/tokenizer.c

 
 /* Tokenizer implementation */
 
+#include "Python.h"
 #include "pgenheaders.h"
 
 #include <ctype.h>
@@ -245,8 +246,8 @@ tok_nextc(register struct tok_state *tok)
 					}
 					tok->end = tok->buf + BUFSIZ;
 				}
-				if (fgets(tok->buf, (int)(tok->end - tok->buf),
-					  tok->fp) == NULL) {
+				if (Py_UniversalNewlineFgets(tok->buf, (int)(tok->end - tok->buf),
+					  tok->fp, NULL) == NULL) {
 					tok->done = E_EOF;
 					done = 1;
 				}
@@ -284,9 +285,9 @@ tok_nextc(register struct tok_state *tok)
 				tok->end = tok->buf + newsize;
 				tok->start = curstart < 0 ? NULL :
 					     tok->buf + curstart;
-				if (fgets(tok->inp,
+				if (Py_UniversalNewlineFgets(tok->inp,
 					       (int)(tok->end - tok->inp),
-					       tok->fp) == NULL) {
+					       tok->fp, NULL) == NULL) {
 					/* Last line does not end in \n,
 					   fake one */
 					strcpy(tok->inp, "\n");

Python/bltinmodule.c

@@ -594,7 +594,7 @@ builtin_execfile(PyObject *self, PyObject *args)
 
         if (exists) {
 		Py_BEGIN_ALLOW_THREADS
-		fp = fopen(filename, "r");
+		fp = fopen(filename, "r" PY_STDIOTEXTMODE);
 		Py_END_ALLOW_THREADS
 
 		if (fp == NULL) {

Python/errors.c

@@ -646,14 +646,14 @@ PyErr_ProgramText(char *filename, int lineno)
 
 	if (filename == NULL || lineno <= 0)
 		return NULL;
-	fp = fopen(filename, "r");
+	fp = fopen(filename, "r" PY_STDIOTEXTMODE);
 	if (fp == NULL)
 		return NULL;
 	for (i = 0; i < lineno; i++) {
 		char *pLastChar = &linebuf[sizeof(linebuf) - 2];
 		do {
 			*pLastChar = '\0';
-			if (fgets(linebuf, sizeof linebuf, fp) == NULL)
+			if (Py_UniversalNewlineFgets(linebuf, sizeof linebuf, fp, NULL) == NULL)
 				break;
 			/* fgets read *something*; if it didn't get as
 			   far as pLastChar, it must have found a newline

Python/import.c

@@ -81,15 +81,15 @@ struct filedescr * _PyImport_Filetab = NULL;
 
 #ifdef RISCOS
 static const struct filedescr _PyImport_StandardFiletab[] = {
-	{"/py", "r", PY_SOURCE},
+	{"/py", "r" PY_STDIOTEXTMODE, PY_SOURCE},
 	{"/pyc", "rb", PY_COMPILED},
 	{0, 0}
 };
 #else
 static const struct filedescr _PyImport_StandardFiletab[] = {
-	{".py", "r", PY_SOURCE},
+	{".py", "r" PY_STDIOTEXTMODE, PY_SOURCE},
 #ifdef MS_WIN32
-	{".pyw", "r", PY_SOURCE},
+	{".pyw", "r" PY_STDIOTEXTMODE, PY_SOURCE},
 #endif
 	{".pyc", "rb", PY_COMPILED},
 	{0, 0}

Python/traceback.c

@@ -155,7 +155,7 @@ tb_displayline(PyObject *f, char *filename, int lineno, char *name)
 	/* This is needed by Emacs' compile command */
 #define FMT "  File \"%.500s\", line %d, in %.500s\n"
 #endif
-	xfp = fopen(filename, "r");
+	xfp = fopen(filename, "r" PY_STDIOTEXTMODE);
 	if (xfp == NULL) {
 		/* Search tail of filename in sys.path before giving up */
 		PyObject *path;
@@ -186,7 +186,7 @@ tb_displayline(PyObject *f, char *filename, int lineno, char *name)
 					if (len > 0 && namebuf[len-1] != SEP)
 						namebuf[len++] = SEP;
 					strcpy(namebuf+len, tail);
-					xfp = fopen(namebuf, "r");
+					xfp = fopen(namebuf, "r" PY_STDIOTEXTMODE);
 					if (xfp != NULL) {
 						filename = namebuf;
 						break;
@@ -203,7 +203,7 @@ tb_displayline(PyObject *f, char *filename, int lineno, char *name)
 		char* pLastChar = &linebuf[sizeof(linebuf)-2];
 		do {
 			*pLastChar = '\0';
-			if (fgets(linebuf, sizeof linebuf, xfp) == NULL)
+			if (Py_UniversalNewlineFgets(linebuf, sizeof linebuf, xfp, NULL) == NULL)
 				break;
 			/* fgets read *something*; if it didn't get as
 			   far as pLastChar, it must have found a newline

README

@@ -811,6 +811,13 @@ Modules/getpath.o.
 --with-pydebug:  Enable additional debugging code to help track down
 	memory management problems.  This allows printing a list of all
 	live objects when the interpreter terminates.
+	
+--with(out)-universal-newlines: enable reading of text files with
+	foreign newline convention (default: enabled). In other words,
+	any of \r, \n or \r\n is acceptable as end-of-line character.
+	If enabled import and execfile will automatically accept any newline
+	in files. Python code can open a file with open(file, 'U') to
+	read it in universal newline mode.
 
 
 Building for multiple architectures (using the VPATH feature)

configure

@@ -844,6 +844,7 @@ Optional Packages:
   --with(out)-thread=DIRECTORY  deprecated; use --with(out)-threads
   --with-pth                      use GNU pth threading libraries
   --with(out)-cycle-gc            disable/enable garbage collection
+  --with(out)-universal-newlines            disable/enable foreign newlines
   --with(out)-pymalloc            disable/enable specialized mallocs
   --with-wctype-functions         use wctype.h functions
   --with-sgi-dl=DIRECTORY         IRIX 4 dynamic linking
@@ -10933,6 +10934,29 @@ fi
 echo "$as_me:$LINENO: result: $with_cycle_gc" >&5
 echo "${ECHO_T}$with_cycle_gc" >&6
 
+# Check for universal newline support
+echo "$as_me:$LINENO: checking for --with-universal-newline" >&5
+echo $ECHO_N "checking for --with-universal-newline... $ECHO_C" >&6
+
+# Check whether --with-universal-newlines or --without-universal-newlines was given.
+if test "${with_universal_newlines+set}" = set; then
+  withval="$with_universal_newlines"
+
+fi;
+
+if test -z "$with_universal_newlines"
+then with_universal_newlines="yes"
+fi
+if test "$with_universal_newlines" != "no"
+then
+    cat >>confdefs.h <<\_ACEOF
+#define WITH_UNIVERSAL_NEWLINES 1
+_ACEOF
+
+fi
+echo "$as_me:$LINENO: result: $with_universal_newlines" >&5
+echo "${ECHO_T}$with_universal_newlines" >&6
+
 # Check for Python-specific malloc support
 echo "$as_me:$LINENO: checking for --with-pymalloc" >&5
 echo $ECHO_N "checking for --with-pymalloc... $ECHO_C" >&6

configure.in

@@ -1445,6 +1445,20 @@ then
 fi
 AC_MSG_RESULT($with_cycle_gc)
 
+# Check for universal newline support
+AC_MSG_CHECKING(for --with-universal-newline)
+AC_ARG_WITH(universal-newlines,
+[  --with(out)-universal-newlines            disable/enable foreign newlines])
+
+if test -z "$with_universal_newlines"
+then with_universal_newlines="yes"
+fi
+if test "$with_universal_newlines" != "no"
+then
+    AC_DEFINE(WITH_UNIVERSAL_NEWLINES)
+fi
+AC_MSG_RESULT($with_universal_newlines)
+
 # Check for Python-specific malloc support
 AC_MSG_CHECKING(for --with-pymalloc)
 AC_ARG_WITH(pymalloc,

pyconfig.h.in

@@ -712,6 +712,9 @@
 /* Define if you want to compile in cycle garbage collection. */
 #undef WITH_CYCLE_GC
 
+/* Define if you want to read files with foreign newlines. */
+#undef WITH_UNIVERSAL_NEWLINES
+
 /* Define if you want to emulate SGI (IRIX 4) dynamic linking. This is
    rumoured to work on VAX (Ultrix), Sun3 (SunOS 3.4), Sequent Symmetry
    (Dynix), and Atari ST. This requires the 'dl-dld' library,