Issue #19544 and Issue #6286: Restore use of urllib over http allowing use of…

Issue #19544 and Issue #6286: Restore use of urllib over http allowing use of http_proxy for Distutils upload command, a feature accidentally lost in the rollback of distutils2.

Lib/distutils/command/upload.py

@@ -10,10 +10,9 @@ import sys
 import os, io
 import socket
 import platform
-import configparser
-import http.client as httpclient
 from base64 import standard_b64encode
-import urllib.parse
+from urllib.request import urlopen, Request, HTTPError
+from urllib.parse import urlparse
 
 # this keeps compatibility for 2.3 and 2.4
 if sys.version < "2.5":
@@ -66,6 +65,15 @@ class upload(PyPIRCCommand):
             self.upload_file(command, pyversion, filename)
 
     def upload_file(self, command, pyversion, filename):
+        # Makes sure the repository URL is compliant
+        schema, netloc, url, params, query, fragments = \
+            urlparse(self.repository)
+        if params or query or fragments:
+            raise AssertionError("Incompatible url %s" % self.repository)
+
+        if schema not in ('http', 'https'):
+            raise AssertionError("unsupported schema " + schema)
+
         # Sign if requested
         if self.sign:
             gpg_args = ["gpg", "--detach-sign", "-a", filename]
@@ -162,41 +170,31 @@ class upload(PyPIRCCommand):
         self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO)
 
         # build the Request
-        # We can't use urllib since we need to send the Basic
-        # auth right with the first request
-        # TODO(jhylton): Can we fix urllib?
-        schema, netloc, url, params, query, fragments = \
-            urllib.parse.urlparse(self.repository)
-        assert not params and not query and not fragments
-        if schema == 'http':
-            http = httpclient.HTTPConnection(netloc)
-        elif schema == 'https':
-            http = httpclient.HTTPSConnection(netloc)
-        else:
-            raise AssertionError("unsupported schema "+schema)
-
-        data = ''
-        loglevel = log.INFO
+        headers = {'Content-type':
+                        'multipart/form-data; boundary=%s' % boundary,
+                   'Content-length': str(len(body)),
+                   'Authorization': auth}
+
+        request = Request(self.repository, data=body,
+                          headers=headers)
+        # send the data
         try:
-            http.connect()
-            http.putrequest("POST", url)
-            http.putheader('Content-type',
-                           'multipart/form-data; boundary=%s'%boundary)
-            http.putheader('Content-length', str(len(body)))
-            http.putheader('Authorization', auth)
-            http.endheaders()
-            http.send(body)
+            result = urlopen(request)
+            status = result.getcode()
+            reason = result.msg
         except socket.error as e:
             self.announce(str(e), log.ERROR)
             return
+        except HTTPError as e:
+            status = e.code
+            reason = e.msg
 
-        r = http.getresponse()
-        if r.status == 200:
-            self.announce('Server response (%s): %s' % (r.status, r.reason),
+        if status == 200:
+            self.announce('Server response (%s): %s' % (status, reason),
                           log.INFO)
         else:
-            self.announce('Upload failed (%s): %s' % (r.status, r.reason),
+            self.announce('Upload failed (%s): %s' % (status, reason),
                           log.ERROR)
         if self.show_response:
-            msg = '\n'.join(('-' * 75, r.read(), '-' * 75))
+            msg = '\n'.join(('-' * 75, result.read(), '-' * 75))
             self.announce(msg, log.INFO)

Lib/distutils/tests/test_upload.py

 """Tests for distutils.command.upload."""
 import os
 import unittest
-import http.client as httpclient
 from test.support import run_unittest
 
+from distutils.command import upload as upload_mod
 from distutils.command.upload import upload
 from distutils.core import Distribution
 
@@ -37,48 +37,37 @@ index-servers =
 [server1]
 username:me
 """
-class Response(object):
-    def __init__(self, status=200, reason='OK'):
-        self.status = status
-        self.reason = reason
 
-class FakeConnection(object):
+class FakeOpen(object):
 
-    def __init__(self):
-        self.requests = []
-        self.headers = []
-        self.body = ''
+    def __init__(self, url):
+        self.url = url
+        if not isinstance(url, str):
+            self.req = url
+        else:
+            self.req = None
+        self.msg = 'OK'
 
-    def __call__(self, netloc):
-        return self
+    def getcode(self):
+        return 200
 
-    def connect(self):
-        pass
-    endheaders = connect
-
-    def putrequest(self, method, url):
-        self.requests.append((method, url))
-
-    def putheader(self, name, value):
-        self.headers.append((name, value))
-
-    def send(self, body):
-        self.body = body
-
-    def getresponse(self):
-        return Response()
 
 class uploadTestCase(PyPIRCCommandTestCase):
 
     def setUp(self):
         super(uploadTestCase, self).setUp()
-        self.old_class = httpclient.HTTPConnection
-        self.conn = httpclient.HTTPConnection = FakeConnection()
+        self.old_open = upload_mod.urlopen
+        upload_mod.urlopen = self._urlopen
+        self.last_open = None
 
     def tearDown(self):
-        httpclient.HTTPConnection = self.old_class
+        upload_mod.urlopen = self.old_open
         super(uploadTestCase, self).tearDown()
 
+    def _urlopen(self, url):
+        self.last_open = FakeOpen(url)
+        return self.last_open
+
     def test_finalize_options(self):
 
         # new format
@@ -122,14 +111,14 @@ class uploadTestCase(PyPIRCCommandTestCase):
         cmd.ensure_finalized()
         cmd.run()
 
-        # what did we send ?
-        headers = dict(self.conn.headers)
+         # what did we send ?
+        headers = dict(self.last_open.req.headers)
         self.assertEqual(headers['Content-length'], '2087')
-        self.assertTrue(headers['Content-type'].startswith('multipart/form-data'))
-        self.assertFalse('\n' in headers['Authorization'])
-
-        self.assertEqual(self.conn.requests, [('POST', '/pypi')])
-        self.assertTrue((b'xxx') in self.conn.body)
+        self.assert_(headers['Content-type'].startswith('multipart/form-data'))
+        self.assertEquals(self.last_open.req.get_method(), 'POST')
+        self.assertEquals(self.last_open.req.get_full_url(),
+                          'http://pypi.python.org/pypi')
+        self.assert_(b'xxx' in self.last_open.req.data)
 
 def test_suite():
     return unittest.makeSuite(uploadTestCase)

Misc/NEWS

@@ -13,6 +13,10 @@ Core and Builtins
 Library
 -------
 
+- Issue #19544 and Issue #6286: Restore use of urllib over http allowing use
+  of http_proxy for Distutils upload command, a feature accidentally lost
+  in the rollback of distutils2.
+
 - Issue #19544 and Issue #7457: Restore the read_pkg_file method to
   distutils.dist.DistributionMetadata accidentally removed in the undo of
   distutils2.