Start improving 2to3 code in packaging (#13462).

- Change the fixers used in tests to something not provided by lib2to3
- Test conversion of doctests in text files
- Factor out test boilerplate into a common method

Lib/packaging/compat.py

-"""Compatibility helpers."""
+"""Support for build-time 2to3 conversion."""
 
 from packaging import logger
 
@@ -25,7 +25,7 @@ class Mixin2to3(_KLASS):
     """
     if _CONVERT:
 
-        def _run_2to3(self, files, doctests=[], fixers=[]):
+        def _run_2to3(self, files=[], doctests=[], fixers=[]):
             """ Takes a list of files and doctests, and performs conversion
             on those.
               - First, the files which contain the code(`files`) are converted.
@@ -35,17 +35,16 @@ class Mixin2to3(_KLASS):
             if fixers:
                 self.fixer_names = fixers
 
-            logger.info('converting Python code')
-            _KLASS.run_2to3(self, files)
+            if files:
+                logger.info('converting Python code and doctests')
+                _KLASS.run_2to3(self, files)
+                _KLASS.run_2to3(self, files, doctests_only=True)
 
-            logger.info('converting doctests in Python files')
-            _KLASS.run_2to3(self, files, doctests_only=True)
-
-            if doctests != []:
-                logger.info('converting doctest in text files')
+            if doctests:
+                logger.info('converting doctests in text files')
                 _KLASS.run_2to3(self, doctests, doctests_only=True)
     else:
         # If run on Python 2.x, there is nothing to do.
 
-        def _run_2to3(self, files, doctests=[], fixers=[]):
+        def _run_2to3(self, files=[], doctests=[], fixers=[]):
             pass

Lib/packaging/tests/fixer/fix_echo.py

+# Example custom fixer, derived from fix_raw_input by Andre Roberge
+
+from lib2to3 import fixer_base
+from lib2to3.fixer_util import Name
+
+
+class FixEcho(fixer_base.BaseFix):
+
+    BM_compatible = True
+    PATTERN = """
+              power< name='echo' trailer< '(' [any] ')' > any* >
+              """
+
+    def transform(self, node, results):
+        name = results['name']
+        name.replace(Name('print', prefix=name.prefix))

Lib/packaging/tests/fixer/fix_echo2.py

+# Example custom fixer, derived from fix_raw_input by Andre Roberge
+
+from lib2to3 import fixer_base
+from lib2to3.fixer_util import Name
+
+
+class FixEcho2(fixer_base.BaseFix):
+
+    BM_compatible = True
+    PATTERN = """
+              power< name='echo2' trailer< '(' [any] ')' > any* >
+              """
+
+    def transform(self, node, results):
+        name = results['name']
+        name.replace(Name('print', prefix=name.prefix))

Lib/packaging/tests/fixer/fix_idioms.py

-"""Adjust some old Python 2 idioms to their modern counterparts.
-
-* Change some type comparisons to isinstance() calls:
-    type(x) == T -> isinstance(x, T)
-    type(x) is T -> isinstance(x, T)
-    type(x) != T -> not isinstance(x, T)
-    type(x) is not T -> not isinstance(x, T)
-
-* Change "while 1:" into "while True:".
-
-* Change both
-
-    v = list(EXPR)
-    v.sort()
-    foo(v)
-
-and the more general
-
-    v = EXPR
-    v.sort()
-    foo(v)
-
-into
-
-    v = sorted(EXPR)
-    foo(v)
-"""
-# Author: Jacques Frechet, Collin Winter
-
-# Local imports
-from lib2to3 import fixer_base
-from lib2to3.fixer_util import Call, Comma, Name, Node, syms
-
-CMP = "(n='!=' | '==' | 'is' | n=comp_op< 'is' 'not' >)"
-TYPE = "power< 'type' trailer< '(' x=any ')' > >"
-
-class FixIdioms(fixer_base.BaseFix):
-
-    explicit = False # The user must ask for this fixer
-
-    PATTERN = r"""
-        isinstance=comparison< %s %s T=any >
-        |
-        isinstance=comparison< T=any %s %s >
-        |
-        while_stmt< 'while' while='1' ':' any+ >
-        |
-        sorted=any<
-            any*
-            simple_stmt<
-              expr_stmt< id1=any '='
-                         power< list='list' trailer< '(' (not arglist<any+>) any ')' > >
-              >
-              '\n'
-            >
-            sort=
-            simple_stmt<
-              power< id2=any
-                     trailer< '.' 'sort' > trailer< '(' ')' >
-              >
-              '\n'
-            >
-            next=any*
-        >
-        |
-        sorted=any<
-            any*
-            simple_stmt< expr_stmt< id1=any '=' expr=any > '\n' >
-            sort=
-            simple_stmt<
-              power< id2=any
-                     trailer< '.' 'sort' > trailer< '(' ')' >
-              >
-              '\n'
-            >
-            next=any*
-        >
-    """ % (TYPE, CMP, CMP, TYPE)
-
-    def match(self, node):
-        r = super(FixIdioms, self).match(node)
-        # If we've matched one of the sort/sorted subpatterns above, we
-        # want to reject matches where the initial assignment and the
-        # subsequent .sort() call involve different identifiers.
-        if r and "sorted" in r:
-            if r["id1"] == r["id2"]:
-                return r
-            return None
-        return r
-
-    def transform(self, node, results):
-        if "isinstance" in results:
-            return self.transform_isinstance(node, results)
-        elif "while" in results:
-            return self.transform_while(node, results)
-        elif "sorted" in results:
-            return self.transform_sort(node, results)
-        else:
-            raise RuntimeError("Invalid match")
-
-    def transform_isinstance(self, node, results):
-        x = results["x"].clone() # The thing inside of type()
-        T = results["T"].clone() # The type being compared against
-        x.prefix = ""
-        T.prefix = " "
-        test = Call(Name("isinstance"), [x, Comma(), T])
-        if "n" in results:
-            test.prefix = " "
-            test = Node(syms.not_test, [Name("not"), test])
-        test.prefix = node.prefix
-        return test
-
-    def transform_while(self, node, results):
-        one = results["while"]
-        one.replace(Name("True", prefix=one.prefix))
-
-    def transform_sort(self, node, results):
-        sort_stmt = results["sort"]
-        next_stmt = results["next"]
-        list_call = results.get("list")
-        simple_expr = results.get("expr")
-
-        if list_call:
-            list_call.replace(Name("sorted", prefix=list_call.prefix))
-        elif simple_expr:
-            new = simple_expr.clone()
-            new.prefix = ""
-            simple_expr.replace(Call(Name("sorted"), [new],
-                                     prefix=simple_expr.prefix))
-        else:
-            raise RuntimeError("should not have reached here")
-        sort_stmt.remove()
-        if next_stmt:
-            next_stmt[0].prefix = sort_stmt._prefix

Lib/packaging/tests/test_mixin2to3.py

@@ -8,70 +8,76 @@ class Mixin2to3TestCase(support.TempdirManager,
                         support.LoggingCatcher,
                         unittest.TestCase):
 
-    @support.skip_2to3_optimize
-    def test_convert_code_only(self):
-        # used to check if code gets converted properly.
-        code = "print 'test'"
+    def setUp(self):
+        super(Mixin2to3TestCase, self).setUp()
+        self.filename = self.mktempfile().name
 
-        with self.mktempfile() as fp:
-            fp.write(code)
+    def check(self, source, wanted, **kwargs):
+        source = textwrap.dedent(source)
+        with open(self.filename, 'w') as fp:
+            fp.write(source)
 
-        mixin2to3 = Mixin2to3()
-        mixin2to3._run_2to3([fp.name])
-        expected = "print('test')"
+        Mixin2to3()._run_2to3(**kwargs)
 
-        with open(fp.name) as fp:
+        wanted = textwrap.dedent(wanted)
+        with open(self.filename) as fp:
             converted = fp.read()
+        self.assertMultiLineEqual(converted, wanted)
 
-        self.assertEqual(expected, converted)
-
-    def test_doctests_only(self):
-        # used to check if doctests gets converted properly.
-        doctest = textwrap.dedent('''\
+    def test_conversion(self):
+        # check that code and doctests get converted
+        self.check('''\
             """Example docstring.
 
             >>> print test
             test
 
             It works.
-            """''')
-
-        with self.mktempfile() as fp:
-            fp.write(doctest)
-
-        mixin2to3 = Mixin2to3()
-        mixin2to3._run_2to3([fp.name])
-        expected = textwrap.dedent('''\
+            """
+            print 'test'
+            ''',
+            '''\
             """Example docstring.
 
             >>> print(test)
             test
 
             It works.
-            """\n''')
+            """
+            print('test')
 
-        with open(fp.name) as fp:
-            converted = fp.read()
-
-        self.assertEqual(expected, converted)
-
-    def test_additional_fixers(self):
-        # used to check if use_2to3_fixers works
-        code = 'type(x) is not T'
+            ''',  # 2to3 adds a newline here
+            files=[self.filename])
 
-        with self.mktempfile() as fp:
-            fp.write(code)
+    def test_doctests_conversion(self):
+        # check that doctest files are converted
+        self.check('''\
+            Welcome to the doc.
 
-        mixin2to3 = Mixin2to3()
-        mixin2to3._run_2to3(files=[fp.name], doctests=[fp.name],
-                            fixers=['packaging.tests.fixer'])
+            >>> print test
+            test
+            ''',
+            '''\
+            Welcome to the doc.
 
-        expected = 'not isinstance(x, T)'
+            >>> print(test)
+            test
 
-        with open(fp.name) as fp:
-            converted = fp.read()
+            ''',
+            doctests=[self.filename])
 
-        self.assertEqual(expected, converted)
+    def test_additional_fixers(self):
+        # make sure the fixers argument works
+        self.check("""\
+            echo('42')
+            echo2('oh no')
+            """,
+            """\
+            print('42')
+            print('oh no')
+            """,
+            files=[self.filename],
+            fixers=['packaging.tests.fixer'])
 
 
 def test_suite():

Lib/packaging/util.py

@@ -853,13 +853,11 @@ def run_2to3(files, doctests_only=False, fixer_names=None,
 
     # Make this class local, to delay import of 2to3
     from lib2to3.refactor import get_fixers_from_package, RefactoringTool
-    fixers = []
     fixers = get_fixers_from_package('lib2to3.fixes')
 
     if fixer_names:
         for fixername in fixer_names:
-            fixers.extend(fixer for fixer in
-                          get_fixers_from_package(fixername))
+            fixers.extend(get_fixers_from_package(fixername))
     r = RefactoringTool(fixers, options=options)
     r.refactor(files, write=True, doctests_only=doctests_only)
 
@@ -870,21 +868,23 @@ class Mixin2to3:
     the class variables, or inherit from this class
     to override how 2to3 is invoked.
     """
-    # provide list of fixers to run.
-    # defaults to all from lib2to3.fixers
+    # list of fixers to run; defaults to all implicit from lib2to3.fixers
     fixer_names = None
-
-    # options dictionary
+    # dict of options
     options = None
-
-    # list of fixers to invoke even though they are marked as explicit
+    # list of extra fixers to invoke
     explicit = None
+    # TODO need a better way to add just one fixer from a package
+    # TODO need a way to exclude individual fixers
 
     def run_2to3(self, files, doctests_only=False):
         """ Issues a call to util.run_2to3. """
         return run_2to3(files, doctests_only, self.fixer_names,
                         self.options, self.explicit)
 
+    # TODO provide initialize/finalize_options
+
+
 RICH_GLOB = re.compile(r'\{([^}]*)\}')
 _CHECK_RECURSIVE_GLOB = re.compile(r'[^/\\,{]\*\*|\*\*[^/\\,}]')
 _CHECK_MISMATCH_SET = re.compile(r'^[^{]*\}|\{[^}]*$')