Convert code from sys.stdin.encoding to UTF-8 in

interactive mode. Fixes #1100.

Include/parsetok.h

@@ -34,7 +34,8 @@ PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int,
 
 PyAPI_FUNC(node *) PyParser_ParseStringFlags(const char *, grammar *, int,
                                               perrdetail *, int);
-PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *, grammar *,
+PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *, 
+					   const char*, grammar *,
 						 int, char *, char *,
 						 perrdetail *, int);
 

Include/pythonrun.h

@@ -40,7 +40,8 @@ PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(FILE *, const char *, PyCompilerFlags
 PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(const char *, const char *, 
 						 int, PyCompilerFlags *flags,
                                                  PyArena *);
-PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *, int, 
+PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *, 
+					       const char*, int, 
 					       char *, char *,
                                                PyCompilerFlags *, int *,
                                                PyArena *);

Parser/parsetok.c

@@ -59,19 +59,20 @@ node *
 PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
 		   char *ps1, char *ps2, perrdetail *err_ret)
 {
-	return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
-				       err_ret, 0);
+	return PyParser_ParseFileFlags(fp, filename, NULL, 
+				       g, start, ps1, ps2, err_ret, 0);
 }
 
 node *
-PyParser_ParseFileFlags(FILE *fp, const char *filename, grammar *g, int start,
+PyParser_ParseFileFlags(FILE *fp, const char *filename, const char* enc,
+			grammar *g, int start,
 			char *ps1, char *ps2, perrdetail *err_ret, int flags)
 {
 	struct tok_state *tok;
 
 	initerr(err_ret, filename);
 
-	if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {
+	if ((tok = PyTokenizer_FromFile(fp, enc, ps1, ps2)) == NULL) {
 		err_ret->error = E_NOMEM;
 		return NULL;
 	}

Parser/tokenizer.c

@@ -677,7 +677,7 @@ PyTokenizer_FromString(const char *str)
 /* Set up tokenizer for file */
 
 struct tok_state *
-PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
+PyTokenizer_FromFile(FILE *fp, char* enc, char *ps1, char *ps2)
 {
 	struct tok_state *tok = tok_new();
 	if (tok == NULL)
@@ -691,6 +691,17 @@ PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
 	tok->fp = fp;
 	tok->prompt = ps1;
 	tok->nextprompt = ps2;
+	if (enc != NULL) {
+		/* Must copy encoding declaration since it
+		   gets copied into the parse tree. */
+		tok->encoding = PyMem_MALLOC(strlen(enc)+1);
+		if (!tok->encoding) {
+			PyTokenizer_Free(tok);
+			return NULL;
+		}
+		strcpy(tok->encoding, enc);
+		tok->decoding_state = -1;
+	}
 	return tok;
 }
 
@@ -742,6 +753,29 @@ tok_nextc(register struct tok_state *tok)
 		}
 		if (tok->prompt != NULL) {
 			char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
+#ifndef PGEN
+			if (tok->encoding && newtok && *newtok) {
+				/* Recode to UTF-8 */
+				Py_ssize_t buflen;
+				const char* buf;
+				PyObject *u = translate_into_utf8(newtok, tok->encoding);
+				PyMem_FREE(newtok);
+				if (!u) {
+					tok->done = E_DECODE;
+					return EOF;
+				}
+				buflen = PyBytes_Size(u);
+				buf = PyBytes_AsString(u);
+				if (!buf) {
+					Py_DECREF(u);
+					tok->done = E_DECODE;
+					return EOF;
+				}
+				newtok = PyMem_MALLOC(buflen+1);
+				strcpy(newtok, buf);
+				Py_DECREF(u);
+			}
+#endif
 			if (tok->nextprompt != NULL)
 				tok->prompt = tok->nextprompt;
 			if (newtok == NULL)

Parser/tokenizer.h

@@ -55,7 +55,8 @@ struct tok_state {
 };
 
 extern struct tok_state *PyTokenizer_FromString(const char *);
-extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *);
+extern struct tok_state *PyTokenizer_FromFile(FILE *, char*,
+					      char *, char *);
 extern void PyTokenizer_Free(struct tok_state *);
 extern int PyTokenizer_Get(struct tok_state *, char **, char **);
 

Python/import.c

@@ -809,7 +809,8 @@ parse_source_module(const char *pathname, FILE *fp)
 	if (arena == NULL)
 		return NULL;
 
-	mod = PyParser_ASTFromFile(fp, pathname, Py_file_input, 0, 0, 0, 
+	mod = PyParser_ASTFromFile(fp, pathname, NULL,
+				   Py_file_input, 0, 0, 0, 
 				   NULL, arena);
 	if (mod) {
 		co = PyAST_Compile(mod, pathname, NULL, arena);

Python/pythonrun.c

@@ -744,12 +744,22 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flag
 int
 PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)
 {
-	PyObject *m, *d, *v, *w;
+	PyObject *m, *d, *v, *w, *oenc = NULL;
 	mod_ty mod;
 	PyArena *arena;
-	char *ps1 = "", *ps2 = "";
+	char *ps1 = "", *ps2 = "", *enc = NULL;
 	int errcode = 0;
 
+	if (fp == stdin) {
+		/* Fetch encoding from sys.stdin */
+		v = PySys_GetObject("stdin");
+		if (!v)
+			return -1;
+		oenc = PyObject_GetAttrString(v, "encoding");
+		if (!oenc)
+			return -1;
+		enc = PyUnicode_AsString(oenc);
+	}
 	v = PySys_GetObject("ps1");
 	if (v != NULL) {
 		v = PyObject_Str(v);
@@ -770,13 +780,15 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags
 	if (arena == NULL) {
 		Py_XDECREF(v);
 		Py_XDECREF(w);
+		Py_XDECREF(oenc);
 		return -1;
 	}
-	mod = PyParser_ASTFromFile(fp, filename,
+	mod = PyParser_ASTFromFile(fp, filename, enc,
 				   Py_single_input, ps1, ps2,
 				   flags, &errcode, arena);
 	Py_XDECREF(v);
 	Py_XDECREF(w);
+	Py_XDECREF(oenc);
 	if (mod == NULL) {
 		PyArena_Free(arena);
 		if (errcode == E_EOF) {
@@ -1254,7 +1266,7 @@ PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals,
 	if (arena == NULL)
 		return NULL;
 	
-	mod = PyParser_ASTFromFile(fp, filename, start, 0, 0,
+	mod = PyParser_ASTFromFile(fp, filename, NULL, start, 0, 0,
 				   flags, NULL, arena);
 	if (closeit)
 		fclose(fp);
@@ -1379,13 +1391,15 @@ PyParser_ASTFromString(const char *s, const char *filename, int start,
 }
 
 mod_ty
-PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1,
+PyParser_ASTFromFile(FILE *fp, const char *filename, const char* enc,
+		     int start, char *ps1,
 		     char *ps2, PyCompilerFlags *flags, int *errcode,
 		     PyArena *arena)
 {
 	mod_ty mod;
 	perrdetail err;
-	node *n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
+	node *n = PyParser_ParseFileFlags(fp, filename, enc,
+					  &_PyParser_Grammar,
 				start, ps1, ps2, &err, PARSER_FLAGS(flags));
 	if (n) {
 		mod = PyAST_FromNode(n, flags, filename, arena);
@@ -1406,7 +1420,8 @@ node *
 PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int flags)
 {
 	perrdetail err;
-	node *n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
+	node *n = PyParser_ParseFileFlags(fp, filename, NULL,
+					  &_PyParser_Grammar,
 					  start, NULL, NULL, &err, flags);
 	if (n == NULL)
 		err_input(&err);