Merge.

Lib/test/support.py

@@ -603,6 +603,32 @@ else:
 # module name.
 TESTFN = "{}_{}_tmp".format(TESTFN, os.getpid())
 
+# FS_NONASCII: non-ASCII character encodable by os.fsencode(),
+# or None if there is no such character.
+FS_NONASCII = None
+for character in (
+    # U+00E6 (Latin small letter AE): Encodable to cp1252, cp1254, cp1257, iso-8859-1
+    '\u00E6',
+    # U+0141 (Latin capital letter L with stroke): Encodable to cp1250, cp1257
+    '\u0141',
+    # U+041A (Cyrillic capital letter KA): Encodable to cp932, cp950, cp1251
+    '\u041A',
+    # U+05D0 (Hebrew Letter Alef): Encodable to cp424, cp1255
+    '\u05D0',
+    # U+06A9 (Arabic letter KEHEH): Encodable to cp1256
+    '\u06A9',
+    # U+03A9 (Greek capital letter OMEGA): Encodable to cp932, cp950, cp1253
+    '\u03A9',
+    # U+0E01 (Thai character KO KAI): Encodable to cp874
+    '\u0E01',
+):
+    try:
+        os.fsdecode(os.fsencode(character))
+    except UnicodeError:
+        pass
+    else:
+        FS_NONASCII = character
+        break
 
 # TESTFN_UNICODE is a non-ascii filename
 TESTFN_UNICODE = TESTFN + "-\xe0\xf2\u0258\u0141\u011f"
@@ -651,13 +677,22 @@ elif sys.platform != 'darwin':
 # decoded from the filesystem encoding (in strict mode). It can be None if we
 # cannot generate such filename.
 TESTFN_UNDECODABLE = None
-for name in (b'abc\xff', b'\xe7w\xf0'):
+# b'\xff' is not decodable by os.fsdecode() with code page 932. Windows
+# accepts it to create a file or a directory, or don't accept to enter to
+# such directory (when the bytes name is used). So test b'\xe7' first: it is
+# not decodable from cp932.
+for name in (b'\xe7w\xf0', b'abc\xff'):
     try:
         os.fsdecode(name)
     except UnicodeDecodeError:
         TESTFN_UNDECODABLE = name
         break
 
+if FS_NONASCII:
+    TESTFN_NONASCII = TESTFN + '- ' + FS_NONASCII
+else:
+    TESTFN_NONASCII = None
+
 # Save the initial cwd
 SAVEDCWD = os.getcwd()
 

Lib/test/test_cmd_line.py

@@ -93,15 +93,15 @@ class CmdLineTest(unittest.TestCase):
         # All good if execution is successful
         assert_python_ok('-c', 'pass')
 
-    @unittest.skipIf(sys.getfilesystemencoding() == 'ascii',
-                     'need a filesystem encoding different than ASCII')
+    @unittest.skipUnless(test.support.FS_NONASCII, 'need support.FS_NONASCII')
     def test_non_ascii(self):
         # Test handling of non-ascii data
         if test.support.verbose:
             import locale
             print('locale encoding = %s, filesystem encoding = %s'
                   % (locale.getpreferredencoding(), sys.getfilesystemencoding()))
-        command = "assert(ord('\xe9') == 0xe9)"
+        command = ("assert(ord(%r) == %s)"
+                   % (test.support.FS_NONASCII, ord(test.support.FS_NONASCII)))
         assert_python_ok('-c', command)
 
     # On Windows, pass bytes to subprocess doesn't test how Python decodes the

Lib/test/test_cmd_line_script.py

@@ -363,19 +363,12 @@ class CmdLineTest(unittest.TestCase):
             self.assertTrue(text[1].startswith('  File '))
             self.assertTrue(text[3].startswith('NameError'))
 
+    @unittest.skipUnless(support.TESTFN_NONASCII, 'need support.TESTFN_NONASCII')
     def test_non_ascii(self):
         # Issue #16218
         # non-ascii filename encodable to cp1252, cp932, latin1 and utf8
-        filename = support.TESTFN + '\xa3'
-        try:
-            os.fsencode(filename)
-        except UnicodeEncodeError:
-            self.skipTest(
-                "Filesystem encoding %r cannot encode "
-                "the filename: %a"
-                % (sys.getfilesystemencoding(), filename))
         source = 'print(ascii(__file__))\n'
-        script_name = _make_test_script(os.curdir, filename, source)
+        script_name = _make_test_script(os.curdir, support.TESTFN_NONASCII, source)
         self.addCleanup(support.unlink, script_name)
         rc, stdout, stderr = assert_python_ok(script_name)
         self.assertEqual(

Lib/test/test_genericpath.py

@@ -308,15 +308,28 @@ class CommonTest(GenericTest):
                 for path in ('', 'fuu', 'f\xf9\xf9', '/fuu', 'U:\\'):
                     self.assertIsInstance(abspath(path), str)
 
-    @unittest.skipIf(sys.platform == 'darwin',
-        "Mac OS X denies the creation of a directory with an invalid utf8 name")
     def test_nonascii_abspath(self):
-        if support.TESTFN_UNDECODABLE:
-            name = support.TESTFN_UNDECODABLE
+        # Test non-ASCII in the path
+        if sys.platform in ('win32', 'darwin'):
+            if support.TESTFN_NONASCII:
+                name = support.TESTFN_NONASCII
+            else:
+                # Mac OS X denies the creation of a directory with an invalid
+                # UTF-8 name. Windows allows to create a directory with an
+                # arbitrary bytes name, but fails to enter this directory
+                # (when the bytes name is used).
+                self.skipTest("need support.TESTFN_NONASCII")
         else:
-            name = b'a\xffb\xe7w\xf0'
+            if support.TESTFN_UNDECODABLE:
+                name = support.TESTFN_UNDECODABLE
+            elif support.TESTFN_NONASCII:
+                name = support.TESTFN_NONASCII
+            else:
+                # On UNIX, the surrogateescape error handler is used to
+                # decode paths, so any byte is allowed, it does not depend
+                # on the locale
+                name = b'a\xffb\xe7w\xf0'
 
-        # Test non-ASCII, non-UTF8 bytes in the path.
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", DeprecationWarning)
             with support.temp_cwd(name):

Lib/test/test_os.py

@@ -1243,6 +1243,8 @@ if sys.platform != 'win32':
         def setUp(self):
             if support.TESTFN_UNENCODABLE:
                 self.dir = support.TESTFN_UNENCODABLE
+            elif support.TESTFN_NONASCII:
+                self.dir = support.TESTFN_NONASCII
             else:
                 self.dir = support.TESTFN
             self.bdir = os.fsencode(self.dir)
@@ -1257,6 +1259,8 @@ if sys.platform != 'win32':
             add_filename(support.TESTFN_UNICODE)
             if support.TESTFN_UNENCODABLE:
                 add_filename(support.TESTFN_UNENCODABLE)
+            if support.TESTFN_NONASCII:
+                add_filename(support.TESTFN_NONASCII)
             if not bytesfn:
                 self.skipTest("couldn't create any non-ascii filename")