Update Demo/parser directory; backport unparse fixes from py3k.

Demo/parser/README

@@ -6,26 +6,27 @@ It also contains examples for the AST parser.
 Files:
 ------
 
-	FILES	     -- list of files associated with the parser module.
+    FILES        -- list of files associated with the parser module.
 
-	README	     -- this file.
+    README       -- this file.
 
-	example.py   --	module that uses the `parser' module to extract
-			information from the parse tree of Python source
-			code.
+    docstring.py -- sample source file containing only a module docstring.
 
-	docstring.py -- sample source file containing only a module docstring.
+    example.py   -- module that uses the `parser' module to extract
+                    information from the parse tree of Python source
+                    code.
 
-	simple.py    -- sample source containing a "short form" definition.
+    simple.py    -- sample source containing a "short form" definition.
 
-	source.py    --	sample source code used to demonstrate ability to
-			handle nested constructs easily using the functions
-			and classes in example.py.
+    source.py    -- sample source code used to demonstrate ability to
+                    handle nested constructs easily using the functions
+                    and classes in example.py.
 
-	test_parser.py  program to put the parser module through its paces.
+    test_parser.py  program to put the parser module through its paces.
 
-	unparse.py	AST (2.5) based example to recreate source code
-			from an AST. This is incomplete; contributions
-			are welcome.
+    test_unparse.py tests for the unparse module
+
+    unparse.py      AST (2.7) based example to recreate source code
+                    from an AST.
 
 Enjoy!

Demo/parser/test_parser.py

@@ -11,19 +11,19 @@ def testChunk(t, fileName):
     global _numFailed
     print '----', fileName,
     try:
-        ast = parser.suite(t)
-        tup = parser.ast2tuple(ast)
-        # this discards the first AST; a huge memory savings when running
+        st = parser.suite(t)
+        tup = parser.st2tuple(st)
+        # this discards the first ST; a huge memory savings when running
         # against a large source file like Tkinter.py.
-        ast = None
-        new = parser.tuple2ast(tup)
+        st = None
+        new = parser.tuple2st(tup)
     except parser.ParserError, err:
         print
         print 'parser module raised exception on input file', fileName + ':'
         traceback.print_exc()
         _numFailed = _numFailed + 1
     else:
-        if tup != parser.ast2tuple(new):
+        if tup != parser.st2tuple(new):
             print
             print 'parser module failed on input file', fileName
             _numFailed = _numFailed + 1

Demo/parser/test_unparse.py

 import unittest
 from test import test_support
-
 import cStringIO
+import sys
+import os
+import tokenize
 import ast
-import _ast
 import unparse
 
-forelse = """\
+def read_pyfile(filename):
+    """Read and return the contents of a Python source file (as a
+    string), taking into account the file encoding."""
+    with open(filename, "r") as pyfile:
+        source = pyfile.read()
+    return source
+
+for_else = """\
 def f():
     for x in range(10):
         break
@@ -15,7 +23,7 @@ def f():
     z = 3
 """
 
-whileelse = """\
+while_else = """\
 def g():
     while True:
         break
@@ -24,16 +32,63 @@ def g():
     z = 3
 """
 
-class UnparseTestCase(unittest.TestCase):
-    # Tests for specific bugs found in earlier versions of unparse
+relative_import = """\
+from . import fred
+from .. import barney
+from .australia import shrimp as prawns
+"""
+
+class_decorator = """\
+@f1(arg)
+@f2
+class Foo: pass
+"""
+
+elif1 = """\
+if cond1:
+    suite1
+elif cond2:
+    suite2
+else:
+    suite3
+"""
+
+elif2 = """\
+if cond1:
+    suite1
+elif cond2:
+    suite2
+"""
+
+try_except_finally = """\
+try:
+    suite1
+except ex1:
+    suite2
+except ex2:
+    suite3
+else:
+    suite4
+finally:
+    suite5
+"""
+
+class ASTTestCase(unittest.TestCase):
+    def assertASTEqual(self, ast1, ast2):
+        dump1 = ast.dump(ast1)
+        dump2 = ast.dump(ast2)
+        self.assertEqual(ast.dump(ast1), ast.dump(ast2))
 
     def check_roundtrip(self, code1, filename="internal"):
-        ast1 = compile(code1, filename, "exec", _ast.PyCF_ONLY_AST)
+        ast1 = compile(code1, filename, "exec", ast.PyCF_ONLY_AST)
         unparse_buffer = cStringIO.StringIO()
         unparse.Unparser(ast1, unparse_buffer)
         code2 = unparse_buffer.getvalue()
-        ast2 = compile(code2, filename, "exec", _ast.PyCF_ONLY_AST)
-        self.assertEqual(ast.dump(ast1), ast.dump(ast2))
+        ast2 = compile(code2, filename, "exec", ast.PyCF_ONLY_AST)
+        self.assertASTEqual(ast1, ast2)
+
+class UnparseTestCase(ASTTestCase):
+    # Tests for specific bugs found in earlier versions of unparse
 
     def test_del_statement(self):
         self.check_roundtrip("del x, y, z")
@@ -43,23 +98,116 @@ class UnparseTestCase(unittest.TestCase):
         self.check_roundtrip("13 >> 7")
 
     def test_for_else(self):
-        self.check_roundtrip(forelse)
+        self.check_roundtrip(for_else)
 
     def test_while_else(self):
-        self.check_roundtrip(whileelse)
+        self.check_roundtrip(while_else)
 
     def test_unary_parens(self):
         self.check_roundtrip("(-1)**7")
+        self.check_roundtrip("(-1.)**8")
+        self.check_roundtrip("(-1j)**6")
         self.check_roundtrip("not True or False")
         self.check_roundtrip("True or not False")
 
+    def test_integer_parens(self):
+        self.check_roundtrip("3 .__abs__()")
+
+    def test_huge_float(self):
+        self.check_roundtrip("1e1000")
+        self.check_roundtrip("-1e1000")
+        self.check_roundtrip("1e1000j")
+        self.check_roundtrip("-1e1000j")
+
+    def test_min_int(self):
+        self.check_roundtrip(str(-sys.maxint-1))
+        self.check_roundtrip("-(%s)" % (sys.maxint + 1))
+
+    def test_imaginary_literals(self):
+        self.check_roundtrip("7j")
+        self.check_roundtrip("-7j")
+        self.check_roundtrip("-(7j)")
+        self.check_roundtrip("0j")
+        self.check_roundtrip("-0j")
+        self.check_roundtrip("-(0j)")
+
+    def test_negative_zero(self):
+        self.check_roundtrip("-0")
+        self.check_roundtrip("-(0)")
+        self.check_roundtrip("-0b0")
+        self.check_roundtrip("-(0b0)")
+        self.check_roundtrip("-0o0")
+        self.check_roundtrip("-(0o0)")
+        self.check_roundtrip("-0x0")
+        self.check_roundtrip("-(0x0)")
+
+    def test_lambda_parentheses(self):
+        self.check_roundtrip("(lambda: int)()")
+
     def test_chained_comparisons(self):
         self.check_roundtrip("1 < 4 <= 5")
         self.check_roundtrip("a is b is c is not d")
 
+    def test_function_arguments(self):
+        self.check_roundtrip("def f(): pass")
+        self.check_roundtrip("def f(a): pass")
+        self.check_roundtrip("def f(b = 2): pass")
+        self.check_roundtrip("def f(a, b): pass")
+        self.check_roundtrip("def f(a, b = 2): pass")
+        self.check_roundtrip("def f(a = 5, b = 2): pass")
+        self.check_roundtrip("def f(*args, **kwargs): pass")
+
+    def test_relative_import(self):
+        self.check_roundtrip(relative_import)
+
+    def test_bytes(self):
+        self.check_roundtrip("b'123'")
+
+    def test_set_literal(self):
+        self.check_roundtrip("{'a', 'b', 'c'}")
+
+    def test_set_comprehension(self):
+        self.check_roundtrip("{x for x in range(5)}")
+
+    def test_dict_comprehension(self):
+        self.check_roundtrip("{x: x*x for x in range(10)}")
+
+    def test_class_decorators(self):
+        self.check_roundtrip(class_decorator)
+
+    def test_elifs(self):
+        self.check_roundtrip(elif1)
+        self.check_roundtrip(elif2)
+
+    def test_try_except_finally(self):
+        self.check_roundtrip(try_except_finally)
+
+class DirectoryTestCase(ASTTestCase):
+    """Test roundtrip behaviour on all files in Lib and Lib/test."""
+
+    # test directories, relative to the root of the distribution
+    test_directories = 'Lib', os.path.join('Lib', 'test')
+
+    def test_files(self):
+        # get names of files to test
+        dist_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
+
+        names = []
+        for d in self.test_directories:
+            test_dir = os.path.join(dist_dir, d)
+            for n in os.listdir(test_dir):
+                if n.endswith('.py') and not n.startswith('bad'):
+                    names.append(os.path.join(test_dir, n))
+
+        for filename in names:
+            if test_support.verbose:
+                print('Testing %s' % filename)
+            source = read_pyfile(filename)
+            self.check_roundtrip(source)
+
 
 def test_main():
-    test_support.run_unittest(UnparseTestCase)
+    test_support.run_unittest(UnparseTestCase, DirectoryTestCase)
 
 if __name__ == '__main__':
     test_main()

Demo/parser/texipre.dat

-\input texinfo   @c -*-texinfo-*-
-@c %**start of header
-@setfilename parser.info
-@settitle Python Parser Module Reference
-@setchapternewpage odd
-@footnotestyle end
-@c %**end of header
-
-@ifinfo
-This file describes the interfaces
-published by the optional @code{parser} module and gives examples of
-how they may be used.  It contains the same text as the chapter on the
-@code{parser} module in the @cite{Python Library Reference}, but is
-presented as a separate document.
-
-Copyright 1995-1996 by Fred L. Drake, Jr., Reston, Virginia, USA, and
-Virginia Polytechnic Institute and State University, Blacksburg,
-Virginia, USA.  Portions of the software copyright 1991-1995 by
-Stichting Mathematisch Centrum, Amsterdam, The Netherlands.  Copying is
-permitted under the terms associated with the main Python distribution,
-with the additional restriction that this additional notice be included
-and maintained on all distributed copies.
-
-                        All Rights Reserved
-
-Permission to use, copy, modify, and distribute this software and its
-documentation for any purpose and without fee is hereby granted,
-provided that the above copyright notice appear in all copies and that
-both that copyright notice and this permission notice appear in
-supporting documentation, and that the names of Fred L. Drake, Jr. and
-Virginia Polytechnic Institute and State University not be used in
-advertising or publicity pertaining to distribution of the software
-without specific, written prior permission.
-
-FRED L. DRAKE, JR. AND VIRGINIA POLYTECHNIC INSTITUTE AND STATE
-UNIVERSITY DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
-INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
-EVENT SHALL FRED L. DRAKE, JR. OR VIRGINIA POLYTECHNIC INSTITUTE AND
-STATE UNIVERSITY BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL
-DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
-PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
-TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-PERFORMANCE OF THIS SOFTWARE.
-@end ifinfo
-
-@titlepage
-@title Python Parser Module Reference
-@author Fred L. Drake, Jr.
-
-@c  The following two commands start the copyright page.
-@page
-@vskip 0pt plus 1filll
-Copyright 1995-1996 by Fred L. Drake, Jr., Reston, Virginia, USA, and
-Virginia Polytechnic Institute and State University, Blacksburg,
-Virginia, USA.  Portions of the software copyright 1991-1995 by
-Stichting Mathematisch Centrum, Amsterdam, The Netherlands.  Copying is
-permitted under the terms associated with the main Python distribution,
-with the additional restriction that this additional notice be included
-and maintained on all distributed copies.
-
-@center All Rights Reserved
-
-Permission to use, copy, modify, and distribute this software and its
-documentation for any purpose and without fee is hereby granted,
-provided that the above copyright notice appear in all copies and that
-both that copyright notice and this permission notice appear in
-supporting documentation, and that the names of Fred L. Drake, Jr. and
-Virginia Polytechnic Institute and State University not be used in
-advertising or publicity pertaining to distribution of the software
-without specific, written prior permission.
-
-FRED L. DRAKE, JR. AND VIRGINIA POLYTECHNIC INSTITUTE AND STATE
-UNIVERSITY DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
-INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
-EVENT SHALL FRED L. DRAKE, JR. OR VIRGINIA POLYTECHNIC INSTITUTE AND
-STATE UNIVERSITY BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL
-DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
-PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
-TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-PERFORMANCE OF THIS SOFTWARE.
-@end titlepage
-
-
-@node Top, Overview, (dir), (dir)
-@top The Python Parser Module
-
-@ifinfo
-This file describes the interfaces
-published by the optional @code{parser} module and gives examples of
-how they may be used.  It contains the same text as the chapter on the
-@code{parser} module in the @cite{Python Library Reference}, but is
-presented as a separate document.
-
-This version corresponds to Python version 1.4 (1 Sept. 1996).
-
-@end ifinfo
-
-@c placeholder for the master menu -- patched by texinfo-all-menus-update
-@menu
-@end menu

Demo/parser/unparse.py

 "Usage: unparse.py <path to source file>"
 import sys
-import _ast
+import ast
 import cStringIO
 import os
 
+# Large float and imaginary literals get turned into infinities in the AST.
+# We unparse those infinities to INFSTR.
+INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1)
+
 def interleave(inter, f, seq):
     """Call f on each item in seq, calling inter() in between.
     """
     seq = iter(seq)
     try:
-        f(seq.next())
+        f(next(seq))
     except StopIteration:
         pass
     else:
@@ -20,15 +24,16 @@ def interleave(inter, f, seq):
 class Unparser:
     """Methods in this class recursively traverse an AST and
     output source code for the abstract syntax; original formatting
-    is disregarged. """
+    is disregarded. """
 
     def __init__(self, tree, file = sys.stdout):
         """Unparser(tree, file=sys.stdout) -> None.
          Print the source for tree to file."""
         self.f = file
+        self.future_imports = []
         self._indent = 0
         self.dispatch(tree)
-        print >>self.f,""
+        self.f.write("")
         self.f.flush()
 
     def fill(self, text = ""):
@@ -79,11 +84,16 @@ class Unparser:
         interleave(lambda: self.write(", "), self.dispatch, t.names)
 
     def _ImportFrom(self, t):
+        # A from __future__ import may affect unparsing, so record it.
+        if t.module and t.module == '__future__':
+            self.future_imports.extend(n.name for n in t.names)
+
         self.fill("from ")
-        self.write(t.module)
+        self.write("." * t.level)
+        if t.module:
+            self.write(t.module)
         self.write(" import ")
         interleave(lambda: self.write(", "), self.dispatch, t.names)
-        # XXX(jpe) what is level for?
 
     def _Assign(self, t):
         self.fill()
@@ -186,10 +196,14 @@ class Unparser:
             self.leave()
 
     def _TryFinally(self, t):
-        self.fill("try")
-        self.enter()
-        self.dispatch(t.body)
-        self.leave()
+        if len(t.body) == 1 and isinstance(t.body[0], ast.TryExcept):
+            # try-except-finally
+            self.dispatch(t.body)
+        else:
+            self.fill("try")
+            self.enter()
+            self.dispatch(t.body)
+            self.leave()
 
         self.fill("finally")
         self.enter()
@@ -202,7 +216,7 @@ class Unparser:
             self.write(" ")
             self.dispatch(t.type)
         if t.name:
-            self.write(", ")
+            self.write(" as ")
             self.dispatch(t.name)
         self.enter()
         self.dispatch(t.body)
@@ -210,6 +224,9 @@ class Unparser:
 
     def _ClassDef(self, t):
         self.write("\n")
+        for deco in t.decorator_list:
+            self.fill("@")
+            self.dispatch(deco)
         self.fill("class "+t.name)
         if t.bases:
             self.write("(")
@@ -251,9 +268,18 @@ class Unparser:
         self.fill("if ")
         self.dispatch(t.test)
         self.enter()
-        # XXX elif?
         self.dispatch(t.body)
         self.leave()
+        # collapse nested ifs into equivalent elifs.
+        while (t.orelse and len(t.orelse) == 1 and
+               isinstance(t.orelse[0], ast.If)):
+            t = t.orelse[0]
+            self.fill("elif ")
+            self.dispatch(t.test)
+            self.enter()
+            self.dispatch(t.body)
+            self.leave()
+        # final else
         if t.orelse:
             self.fill("else")
             self.enter()
@@ -284,7 +310,17 @@ class Unparser:
 
     # expr
     def _Str(self, tree):
-        self.write(repr(tree.s))
+        # if from __future__ import unicode_literals is in effect,
+        # then we want to output string literals using a 'b' prefix
+        # and unicode literals with no prefix.
+        if "unicode_literals" not in self.future_imports:
+            self.write(repr(tree.s))
+        elif isinstance(tree.s, str):
+            self.write("b" + repr(tree.s))
+        elif isinstance(tree.s, unicode):
+            self.write(repr(tree.s).lstrip("u"))
+        else:
+            assert False, "shouldn't get here"
 
     def _Name(self, t):
         self.write(t.id)
@@ -295,16 +331,14 @@ class Unparser:
         self.write("`")
 
     def _Num(self, t):
-        # There are no negative numeric literals in Python; however,
-        # some optimizations produce a negative Num in the AST. Add
-        # parentheses to avoid turning (-1)**2 into -1**2.
-        strnum = repr(t.n)
-        if strnum.startswith("-"):
+        repr_n = repr(t.n)
+        # Parenthesize negative numbers, to avoid turning (-1)**2 into -1**2.
+        if repr_n.startswith("-"):
             self.write("(")
-            self.write(strnum)
+        # Substitute overflowing decimal literal for AST infinities.
+        self.write(repr_n.replace("inf", INFSTR))
+        if repr_n.startswith("-"):
             self.write(")")
-        else:
-            self.write(repr(t.n))
 
     def _List(self, t):
         self.write("[")
@@ -325,6 +359,22 @@ class Unparser:
             self.dispatch(gen)
         self.write(")")
 
+    def _SetComp(self, t):
+        self.write("{")
+        self.dispatch(t.elt)
+        for gen in t.generators:
+            self.dispatch(gen)
+        self.write("}")
+
+    def _DictComp(self, t):
+        self.write("{")
+        self.dispatch(t.key)
+        self.write(": ")
+        self.dispatch(t.value)
+        for gen in t.generators:
+            self.dispatch(gen)
+        self.write("}")
+
     def _comprehension(self, t):
         self.write(" for ")
         self.dispatch(t.target)
@@ -343,13 +393,20 @@ class Unparser:
         self.dispatch(t.orelse)
         self.write(")")
 
+    def _Set(self, t):
+        assert(t.elts) # should be at least one element
+        self.write("{")
+        interleave(lambda: self.write(", "), self.dispatch, t.elts)
+        self.write("}")
+
     def _Dict(self, t):
         self.write("{")
-        def writem((k, v)):
+        def write_pair(pair):
+            (k, v) = pair
             self.dispatch(k)
             self.write(": ")
             self.dispatch(v)
-        interleave(lambda: self.write(", "), writem, zip(t.keys, t.values))
+        interleave(lambda: self.write(", "), write_pair, zip(t.keys, t.values))
         self.write("}")
 
     def _Tuple(self, t):
@@ -367,7 +424,17 @@ class Unparser:
         self.write("(")
         self.write(self.unop[t.op.__class__.__name__])
         self.write(" ")
-        self.dispatch(t.operand)
+        # If we're applying unary minus to a number, parenthesize the number.
+        # This is necessary: -2147483648 is different from -(2147483648) on
+        # a 32-bit machine (the first is an int, the second a long), and
+        # -7j is different from -(7j).  (The first has real part 0.0, the second
+        # has real part -0.0.)
+        if isinstance(t.op, ast.USub) and isinstance(t.operand, ast.Num):
+            self.write("(")
+            self.dispatch(t.operand)
+            self.write(")")
+        else:
+            self.dispatch(t.operand)
         self.write(")")
 
     binop = { "Add":"+", "Sub":"-", "Mult":"*", "Div":"/", "Mod":"%",
@@ -390,7 +457,7 @@ class Unparser:
             self.dispatch(e)
         self.write(")")
 
-    boolops = {_ast.And: 'and', _ast.Or: 'or'}
+    boolops = {ast.And: 'and', ast.Or: 'or'}
     def _BoolOp(self, t):
         self.write("(")
         s = " %s " % self.boolops[t.op.__class__]
@@ -399,6 +466,11 @@ class Unparser:
 
     def _Attribute(self,t):
         self.dispatch(t.value)
+        # Special case: 3.__abs__() is a syntax error, so if t.value
+        # is an integer literal then we need to either parenthesize
+        # it or add an extra space to get 3 .__abs__().
+        if isinstance(t.value, ast.Num) and isinstance(t.value.n, int):
+            self.write(" ")
         self.write(".")
         self.write(t.attr)
 
@@ -455,21 +527,24 @@ class Unparser:
     # others
     def _arguments(self, t):
         first = True
-        nonDef = len(t.args)-len(t.defaults)
-        for a in t.args[0:nonDef]:
-            if first:first = False
-            else: self.write(", ")
-            self.dispatch(a)
-        for a,d in zip(t.args[nonDef:], t.defaults):
+        # normal arguments
+        defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults
+        for a,d in zip(t.args, defaults):
             if first:first = False
             else: self.write(", ")
             self.dispatch(a),
-            self.write("=")
-            self.dispatch(d)
+            if d:
+                self.write("=")
+                self.dispatch(d)
+
+        # varargs
         if t.vararg:
             if first:first = False
             else: self.write(", ")
-            self.write("*"+t.vararg)
+            self.write("*")
+            self.write(t.vararg)
+
+        # kwargs
         if t.kwarg:
             if first:first = False
             else: self.write(", ")
@@ -481,10 +556,12 @@ class Unparser:
         self.dispatch(t.value)
 
     def _Lambda(self, t):
+        self.write("(")
         self.write("lambda ")
         self.dispatch(t.args)
         self.write(": ")
         self.dispatch(t.body)
+        self.write(")")
 
     def _alias(self, t):
         self.write(t.name)
@@ -492,8 +569,9 @@ class Unparser:
             self.write(" as "+t.asname)
 
 def roundtrip(filename, output=sys.stdout):
-    source = open(filename).read()
-    tree = compile(source, filename, "exec", _ast.PyCF_ONLY_AST)
+    with open(filename, "r") as pyfile:
+        source = pyfile.read()
+    tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST)
     Unparser(tree, output)
 
 
@@ -502,7 +580,7 @@ def testdir(a):
     try:
         names = [n for n in os.listdir(a) if n.endswith('.py')]
     except OSError:
-        print >> sys.stderr, "Directory not readable: %s" % a
+        sys.stderr.write("Directory not readable: %s" % a)
     else:
         for n in names:
             fullname = os.path.join(a, n)
@@ -511,7 +589,7 @@ def testdir(a):
                 print 'Testing %s' % fullname
                 try:
                     roundtrip(fullname, output)
-                except Exception, e:
+                except Exception as e:
                     print '  Failed to compile, exception is %s' % repr(e)
             elif os.path.isdir(fullname):
                 testdir(fullname)