Fix Unicode issue #26 for Python 3 files. (#77)

* Fix Unicode issue #26 for Python 3 files.

astor/file_util.py

@@ -15,6 +15,11 @@ import ast
 import sys
 import os
 
+try:
+    from tokenize import open as fopen
+except ImportError:
+    fopen = open
+
 
 class CodeToAst(object):
     """Given a module, or a function that was compiled as part
@@ -52,10 +57,11 @@ class CodeToAst(object):
 
         This is a very thin wrapper around ast.parse
 
-            TODO: Handle encodings other than the default (issue #26)
+            TODO: Handle encodings other than the default for Python 2
+                        (issue #26)
         """
         try:
-            with open(fname, 'r') as f:
+            with fopen(fname) as f:
                 fstr = f.read()
         except IOError:
             if fname != 'stdin':

astor/rtrip.py

@@ -84,6 +84,11 @@ from astor.node_util import (allow_ast_comparison, dump_tree,
 
 dsttree = 'tmp_rtrip'
 
+# TODO:  Remove this workaround once we remove version 2 support
+
+def out_prep(s, pre_encoded=(sys.version_info[0] == 2)):
+    return s if pre_encoded else s.encode('utf-8')
+
 
 def convert(srctree, dsttree=dsttree, readonly=False, dumpall=False,
             ignore_exceptions=False, fullcomp=False):
@@ -147,8 +152,8 @@ def convert(srctree, dsttree=dsttree, readonly=False, dumpall=False,
         if not readonly:
             dstfname = os.path.join(dstpath, fname)
             try:
-                with open(dstfname, 'w') as f:
-                    f.write(dsttxt)
+                with open(dstfname, 'wb') as f:
+                    f.write(out_prep(dsttxt))
             except UnicodeEncodeError:
                 badfiles.add(dstfname)
 
@@ -174,13 +179,13 @@ def convert(srctree, dsttree=dsttree, readonly=False, dumpall=False,
             if dumpall or bad:
                 if not readonly:
                     try:
-                        with open(dstfname[:-3] + '.srcdmp', 'w') as f:
-                            f.write(srcdump)
+                        with open(dstfname[:-3] + '.srcdmp', 'wb') as f:
+                            f.write(out_prep(srcdump))
                     except UnicodeEncodeError:
                         badfiles.add(dstfname[:-3] + '.srcdmp')
                     try:
-                        with open(dstfname[:-3] + '.dstdmp', 'w') as f:
-                            f.write(dstdump)
+                        with open(dstfname[:-3] + '.dstdmp', 'wb') as f:
+                            f.write(out_prep(dstdump))
                     except UnicodeEncodeError:
                         badfiles.add(dstfname[:-3] + '.dstdmp')
                 elif dumpall:
@@ -208,6 +213,7 @@ def convert(srctree, dsttree=dsttree, readonly=False, dumpall=False,
     if bad_nodes:
         logging.error('\nERROR -- UNKNOWN NODES STRIPPED: %s' % bad_nodes)
     logging.info('\n')
+    return broken
 
 
 def usage(msg):

setup.py

@@ -27,9 +27,10 @@ setup(
         'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.2',
         'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: Implementation',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',

tests/test_rtrip.py

+"""
+Part of the astor library for Python AST manipulation
+
+License: 3-clause BSD
+
+Copyright (c) 2017 Patrick Maupin
+"""
+
+import os
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
+
+import astor.rtrip
+
+
+class RtripTestCase(unittest.TestCase):
+
+    def test_convert_stdlib(self):
+        srcdir = os.path.dirname(os.__file__)
+        result = astor.rtrip.convert(srcdir)
+        self.assertEqual(result, [])
+
+
+if __name__ == '__main__':
+    unittest.main()

tox.ini

 [tox]
-envlist = py26, py27, py33, py34, pypy, pypy3.3-5.2-alpha1
+envlist = py26, py27, py33, py34, py35, py36, pypy, pypy3.3-5.2-alpha1
 skipsdist = True
 
 [testenv]