Merged revisions 62004 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r62004 | georg.brandl | 2008-03-28 13:11:56 +0100 (Fr, 28 Mär 2008) | 4 lines

  Patch #1810 by Thomas Lee, reviewed by myself:
  allow compiling Python AST objects into code objects
  in compile().
........

Doc/library/_ast.rst

@@ -10,16 +10,16 @@ Abstract Syntax Trees
 
 
 The ``_ast`` module helps Python applications to process trees of the Python
-abstract syntax grammar. The Python compiler currently provides read-only access
-to such trees, meaning that applications can only create a tree for a given
-piece of Python source code; generating :term:`bytecode` from a (potentially modified)
-tree is not supported. The abstract syntax itself might change with each Python
-release; this module helps to find out programmatically what the current grammar
-looks like.
+abstract syntax grammar.  The abstract syntax itself might change with each
+Python release; this module helps to find out programmatically what the current
+grammar looks like.
 
-An abstract syntax tree can be generated by passing ``_ast.PyCF_ONLY_AST`` as a
-flag to the :func:`compile` builtin function. The result will be a tree of
-objects whose classes all inherit from ``_ast.AST``.
+An abstract syntax tree can be generated by passing :data:`_ast.PyCF_ONLY_AST`
+as a flag to the :func:`compile` builtin function. The result will be a tree of
+objects whose classes all inherit from :class:`_ast.AST`.
+
+A modified abstract syntax tree can be compiled into a Python code object using
+the built-in :func:`compile` function.
 
 The actual classes are derived from the ``Parser/Python.asdl`` file, which is
 reproduced below. There is one class defined for each left-hand side symbol in
@@ -39,12 +39,15 @@ attribute ``left`` of type ``_ast.expr``.   Instances of ``_ast.expr`` and
 ``_ast.stmt`` subclasses also have lineno and col_offset attributes.  The lineno
 is the line number of source text (1 indexed so the first line is line 1) and
 the col_offset is the utf8 byte offset of the first token that generated the
-node.  The utf8 offset is recorded because the parser uses utf8  internally.
+node.  The utf8 offset is recorded because the parser uses utf8 internally.
 
 If these attributes are marked as optional in the grammar (using a question
 mark), the value might be ``None``. If the attributes can have zero-or-more
 values (marked with an asterisk), the values are represented as Python lists.
 
+The constructors of all ``_ast`` classes don't take arguments; instead, if you
+create instances, you must assign the required attributes separately.
+
 
 Abstract Grammar
 ----------------

Doc/library/functions.rst

@@ -193,21 +193,21 @@ available.  They are listed here in alphabetical order.
 
 .. function:: compile(source, filename, mode[, flags[, dont_inherit]])
 
-   Compile the *source* into a code object.  Code objects can be executed by a call
-   to :func:`exec` or evaluated by a call to :func:`eval`.  The *filename* argument
-   should give the file from which the code was read; pass some recognizable value
-   if it wasn't read from a file (``'<string>'`` is commonly used). The *mode*
-   argument specifies what kind of code must be compiled; it can be ``'exec'`` if
-   *source* consists of a sequence of statements, ``'eval'`` if it consists of a
-   single expression, or ``'single'`` if it consists of a single interactive
-   statement (in the latter case, expression statements that evaluate to something
-   else than ``None`` will be printed).
-
-   When compiling multi-line statements, two caveats apply: line endings must be
-   represented by a single newline character (``'\n'``), and the input must be
-   terminated by at least one newline character.  If line endings are represented
-   by ``'\r\n'``, use the string :meth:`replace` method to change them into
-   ``'\n'``.
+   Compile the *source* into a code object.  Code objects can be
+   executed by a call to :func:`exec` or evaluated by a call to
+   :func:`eval`. *source* can either be a string or an AST object.
+   Refer to the :mod:`_ast` module documentation for information on
+   how to compile into and from AST objects.
+
+   The *filename* argument should give the file from
+   which the code was read; pass some recognizable value if it wasn't
+   read from a file (``'<string>'`` is commonly used). The *mode*
+   argument specifies what kind of code must be compiled; it can be
+   ``'exec'`` if *source* consists of a sequence of statements,
+   ``'eval'`` if it consists of a single expression, or ``'single'``
+   if it consists of a single interactive statement (in the latter
+   case, expression statements that evaluate to something else than
+   ``None`` will be printed).
 
    The optional arguments *flags* and *dont_inherit* (which are new in Python 2.2)
    control which future statements (see :pep:`236`) affect the compilation of
@@ -227,6 +227,9 @@ available.  They are listed here in alphabetical order.
    This function raises :exc:`SyntaxError` if the compiled source is invalid,
    and :exc:`TypeError` if the source contains null bytes.
 
+   .. versionadded:: 2.6
+      Support for compiling AST objects.
+
 
 .. function:: complex([real[, imag]])
 

Include/Python-ast.h

@@ -542,3 +542,5 @@ keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena);
 alias_ty _Py_alias(identifier name, identifier asname, PyArena *arena);
 
 PyObject* PyAST_mod2obj(mod_ty t);
+mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena);
+int PyAST_Check(PyObject* obj);

Lib/test/test_compile.py

 import unittest
 import sys
+import _ast
 from test import test_support
 
 class TestSpecifics(unittest.TestCase):
@@ -406,6 +407,28 @@ if 1:
         self.assert_("_A__mangled_mod" in A.f.__code__.co_varnames)
         self.assert_("__package__" in A.f.__code__.co_varnames)
 
+    def test_compile_ast(self):
+        fname = __file__
+        if fname.lower().endswith(('pyc', 'pyo')):
+            fname = fname[:-1]
+        with open(fname, 'r') as f:
+            fcontents = f.read()
+        sample_code = [
+            ['<assign>', 'x = 5'],
+            ['<ifblock>', """if True:\n    pass\n"""],
+            ['<forblock>', """for n in [1, 2, 3]:\n    print(n)\n"""],
+            ['<deffunc>', """def foo():\n    pass\nfoo()\n"""],
+            [fname, fcontents],
+        ]
+
+        for fname, code in sample_code:
+            co1 = compile(code, '%s1' % fname, 'exec')
+            ast = compile(code, '%s2' % fname, 'exec', _ast.PyCF_ONLY_AST)
+            self.assert_(type(ast) == _ast.Module)
+            co2 = compile(ast, '%s3' % fname, 'exec')
+            self.assertEqual(co1, co2)
+
+
 def test_main():
     test_support.run_unittest(TestSpecifics)
 

Python/bltinmodule.c

 /* Built-in functions */
 
 #include "Python.h"
+#include "Python-ast.h"
 
 #include "node.h"
 #include "code.h"
@@ -527,10 +528,43 @@ builtin_compile(PyObject *self, PyObject *args, PyObject *kwds)
 
 	cf.cf_flags = supplied_flags | PyCF_SOURCE_IS_UTF8;
 
-	str = source_as_string(cmd);
-	if (str == NULL)
+	if (supplied_flags &
+	    ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT | PyCF_ONLY_AST))
+	{
+		PyErr_SetString(PyExc_ValueError,
+				"compile(): unrecognised flags");
 		return NULL;
+	}
+	/* XXX Warn if (supplied_flags & PyCF_MASK_OBSOLETE) != 0? */
 
+	if (!dont_inherit) {
+		PyEval_MergeCompilerFlags(&cf);
+	}
+
+	if (PyAST_Check(cmd)) {
+		PyObject *result;
+		if (supplied_flags & PyCF_ONLY_AST) {
+			Py_INCREF(cmd);
+			result = cmd;
+		}
+		else {
+			PyArena *arena;
+			mod_ty mod;
+
+			arena = PyArena_New();
+			mod = PyAST_obj2mod(cmd, arena);
+			if (mod == NULL) {
+				PyArena_Free(arena);
+				return NULL;
+			}
+			result = (PyObject*)PyAST_Compile(mod, filename,
+							  &cf, arena);
+			PyArena_Free(arena);
+		}
+		return result;
+	}
+
+	/* XXX: is it possible to pass start to the PyAST_ branch? */
 	if (strcmp(startstr, "exec") == 0)
 		start = Py_file_input;
 	else if (strcmp(startstr, "eval") == 0)
@@ -539,22 +573,15 @@ builtin_compile(PyObject *self, PyObject *args, PyObject *kwds)
 		start = Py_single_input;
 	else {
 		PyErr_SetString(PyExc_ValueError,
-		   "compile() arg 3 must be 'exec' or 'eval' or 'single'");
+				"compile() arg 3 must be 'exec'"
+				"or 'eval' or 'single'");
 		return NULL;
 	}
 
-	if (supplied_flags &
-	    ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT | PyCF_ONLY_AST))
-	{
-		PyErr_SetString(PyExc_ValueError,
-				"compile(): unrecognised flags");
+	str = source_as_string(cmd);
+	if (str == NULL)
 		return NULL;
-	}
-	/* XXX Warn if (supplied_flags & PyCF_MASK_OBSOLETE) != 0? */
 
-	if (!dont_inherit) {
-		PyEval_MergeCompilerFlags(&cf);
-	}
 	return Py_CompileStringFlags(str, filename, start, &cf);
 }
 

Python/compile.c

@@ -2356,8 +2356,11 @@ unaryop(unaryop_ty op)
 		return UNARY_POSITIVE;
 	case USub:
 		return UNARY_NEGATIVE;
+	default:
+		PyErr_Format(PyExc_SystemError,
+			"unary op %d should not be possible", op);
+		return 0;
 	}
-	return 0;
 }
 
 static int
@@ -2388,8 +2391,11 @@ binop(struct compiler *c, operator_ty op)
 		return BINARY_AND;
 	case FloorDiv:
 		return BINARY_FLOOR_DIVIDE;
+	default:
+		PyErr_Format(PyExc_SystemError,
+			"binary op %d should not be possible", op);
+		return 0;
 	}
-	return 0;
 }
 
 static int
@@ -2416,8 +2422,9 @@ cmpop(cmpop_ty op)
 		return PyCmp_IN;
 	case NotIn:
 		return PyCmp_NOT_IN;
+	default:
+		return PyCmp_BAD;
 	}
-	return PyCmp_BAD;
 }
 
 static int
@@ -2448,10 +2455,11 @@ inplace_binop(struct compiler *c, operator_ty op)
 		return INPLACE_AND;
 	case FloorDiv:
 		return INPLACE_FLOOR_DIVIDE;
+	default:
+		PyErr_Format(PyExc_SystemError,
+			"inplace binary op %d should not be possible", op);
+		return 0;
 	}
-	PyErr_Format(PyExc_SystemError,
-		     "inplace binary op %d should not be possible", op);
-	return 0;
 }
 
 static int