Issue #12112: fix the encoding of setup.py in the packaging module

 * read: use tokenize.detect_encoding()
 * write: use 'utf-8'

Lib/packaging/create.py

@@ -32,6 +32,7 @@ import glob
 import re
 import shutil
 import sysconfig
+import tokenize
 from configparser import RawConfigParser
 from textwrap import dedent
 from hashlib import md5
@@ -116,7 +117,9 @@ def load_setup():
     This function load the setup file in all cases (even if it have already
     been loaded before, because we are monkey patching its setup function with
     a particular one"""
-    with open("setup.py") as f:
+    with open("setup.py", "rb") as f:
+        encoding, lines = tokenize.detect_encoding(f.readline)
+    with open("setup.py", encoding=encoding) as f:
         imp.load_module("setup", f, "setup.py", (".py", "r", imp.PY_SOURCE))
 
 

Lib/packaging/util.py

@@ -346,9 +346,9 @@ def byte_compile(py_files, optimize=0, force=False, prefix=None,
         logger.info("writing byte-compilation script '%s'", script_name)
         if not dry_run:
             if script_fd is not None:
-                script = os.fdopen(script_fd, "w")
+                script = os.fdopen(script_fd, "w", encoding='utf-8')
             else:
-                script = open(script_name, "w")
+                script = open(script_name, "w", encoding='utf-8')
 
             with script:
                 script.write("""\
@@ -1087,7 +1087,7 @@ def generate_setup_py():
     if os.path.exists("setup.py"):
         raise PackagingFileError("a setup.py file alreadyexists")
 
-    with open("setup.py", "w") as fp:
+    with open("setup.py", "w", encoding='utf-8') as fp:
         fp.write(_SETUP_TMPL % {'func': getsource(cfg_to_args)})