test_urllib2net.py 12 KB
Newer Older
1
import unittest
2
from test import support
3
from test.test_urllib2 import sanepathname2url
4

5
import os
6
import socket
7 8
import urllib.error
import urllib.request
9
import sys
10

11 12
support.requires("network")

13
TIMEOUT = 60  # seconds
14

15

Georg Brandl's avatar
Georg Brandl committed
16
def _retry_thrice(func, exc, *args, **kwargs):
17 18
    for i in range(3):
        try:
Georg Brandl's avatar
Georg Brandl committed
19 20
            return func(*args, **kwargs)
        except exc as e:
Neal Norwitz's avatar
Neal Norwitz committed
21
            last_exc = e
22 23 24
            continue
    raise last_exc

Georg Brandl's avatar
Georg Brandl committed
25 26 27 28 29 30 31
def _wrap_with_retry_thrice(func, exc):
    def wrapped(*args, **kwargs):
        return _retry_thrice(func, exc, *args, **kwargs)
    return wrapped

# Connecting to remote hosts is flaky.  Make it more robust by retrying
# the connection several times.
32 33
_urlopen_with_retry = _wrap_with_retry_thrice(urllib.request.urlopen,
                                              urllib.error.URLError)
34

35 36 37 38 39 40 41 42

class AuthTests(unittest.TestCase):
    """Tests urllib2 authentication features."""

## Disabled at the moment since there is no page under python.org which
## could be used to HTTP authentication.
#
#    def test_basic_auth(self):
43
#        import http.client
44 45 46 47 48 49 50 51 52
#
#        test_url = "http://www.python.org/test/test_urllib2/basic_auth"
#        test_hostport = "www.python.org"
#        test_realm = 'Test Realm'
#        test_user = 'test.test_urllib2net'
#        test_password = 'blah'
#
#        # failure
#        try:
53
#            _urlopen_with_retry(test_url)
54 55 56 57 58 59 60 61 62 63 64
#        except urllib2.HTTPError, exc:
#            self.assertEqual(exc.code, 401)
#        else:
#            self.fail("urlopen() should have failed with 401")
#
#        # success
#        auth_handler = urllib2.HTTPBasicAuthHandler()
#        auth_handler.add_password(test_realm, test_hostport,
#                                  test_user, test_password)
#        opener = urllib2.build_opener(auth_handler)
#        f = opener.open('http://localhost/')
65
#        response = _urlopen_with_retry("http://www.python.org/")
66 67 68 69 70
#
#        # The 'userinfo' URL component is deprecated by RFC 3986 for security
#        # reasons, let's not implement it!  (it's already implemented for proxy
#        # specification strings (that is, URLs or authorities specifying a
#        # proxy), so we must keep that)
71
#        self.assertRaises(http.client.InvalidURL,
72 73 74
#                          urllib2.urlopen, "http://evil:thing@example.com")


75 76 77 78 79
class CloseSocketTest(unittest.TestCase):

    def test_close(self):
        # calling .close() on urllib2's response objects should close the
        # underlying socket
80
        url = "http://www.example.com/"
81 82 83
        with support.transient_internet(url):
            response = _urlopen_with_retry(url)
            sock = response.fp
84
            self.assertFalse(sock.closed)
85 86
            response.close()
            self.assertTrue(sock.closed)
87

88 89 90 91 92 93 94 95 96 97 98 99
class OtherNetworkTests(unittest.TestCase):
    def setUp(self):
        if 0:  # for debugging
            import logging
            logger = logging.getLogger("test_urllib2net")
            logger.addHandler(logging.StreamHandler())

    # XXX The rest of these tests aren't very good -- they don't check much.
    # They do sometimes catch some major disasters, though.

    def test_ftp(self):
        urls = [
100 101
            'ftp://www.pythontest.net/README',
            ('ftp://www.pythontest.net/non-existent-file',
102
             None, urllib.error.URLError),
103 104 105 106
            ]
        self._test_urls(urls, self._extra_handlers())

    def test_file(self):
107
        TESTFN = support.TESTFN
108 109 110 111 112
        f = open(TESTFN, 'w')
        try:
            f.write('hi there\n')
            f.close()
            urls = [
113 114 115
                'file:' + sanepathname2url(os.path.abspath(TESTFN)),
                ('file:///nonsensename/etc/passwd', None,
                 urllib.error.URLError),
116
                ]
Georg Brandl's avatar
Georg Brandl committed
117
            self._test_urls(urls, self._extra_handlers(), retry=True)
118 119 120
        finally:
            os.remove(TESTFN)

121 122
        self.assertRaises(ValueError, urllib.request.urlopen,'./relative_path/to/file')

123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150
    # XXX Following test depends on machine configurations that are internal
    # to CNRI.  Need to set up a public server with the right authentication
    # configuration for test purposes.

##     def test_cnri(self):
##         if socket.gethostname() == 'bitdiddle':
##             localhost = 'bitdiddle.cnri.reston.va.us'
##         elif socket.gethostname() == 'bitdiddle.concentric.net':
##             localhost = 'localhost'
##         else:
##             localhost = None
##         if localhost is not None:
##             urls = [
##                 'file://%s/etc/passwd' % localhost,
##                 'http://%s/simple/' % localhost,
##                 'http://%s/digest/' % localhost,
##                 'http://%s/not/found.h' % localhost,
##                 ]

##             bauth = HTTPBasicAuthHandler()
##             bauth.add_password('basic_test_realm', localhost, 'jhylton',
##                                'password')
##             dauth = HTTPDigestAuthHandler()
##             dauth.add_password('digest_test_realm', localhost, 'jhylton',
##                                'password')

##             self._test_urls(urls, self._extra_handlers()+[bauth, dauth])

151
    def test_urlwithfrag(self):
152
        urlwith_frag = "http://www.pythontest.net/index.html#frag"
153 154 155 156
        with support.transient_internet(urlwith_frag):
            req = urllib.request.Request(urlwith_frag)
            res = urllib.request.urlopen(req)
            self.assertEqual(res.geturl(),
157
                    "http://www.pythontest.net/index.html#frag")
158

159
    def test_redirect_url_withfrag(self):
160
        redirect_url_with_frag = "http://www.pythontest.net/redir/with_frag/"
161 162 163 164
        with support.transient_internet(redirect_url_with_frag):
            req = urllib.request.Request(redirect_url_with_frag)
            res = urllib.request.urlopen(req)
            self.assertEqual(res.geturl(),
165
                    "http://www.pythontest.net/elsewhere/#frag")
166

167 168
    def test_custom_headers(self):
        url = "http://www.example.com"
169 170 171 172 173 174 175 176 177 178
        with support.transient_internet(url):
            opener = urllib.request.build_opener()
            request = urllib.request.Request(url)
            self.assertFalse(request.header_items())
            opener.open(request)
            self.assertTrue(request.header_items())
            self.assertTrue(request.has_header('User-agent'))
            request.add_header('User-Agent','Test-Agent')
            opener.open(request)
            self.assertEqual(request.get_header('User-agent'),'Test-Agent')
179

180
    @unittest.skip('XXX: http://www.imdb.com is gone')
181 182 183 184
    def test_sites_no_connection_close(self):
        # Some sites do not send Connection: close header.
        # Verify that those work properly. (#issue12576)

185 186 187 188 189 190 191 192 193 194 195 196 197 198
        URL = 'http://www.imdb.com' # mangles Connection:close

        with support.transient_internet(URL):
            try:
                with urllib.request.urlopen(URL) as res:
                    pass
            except ValueError as e:
                self.fail("urlopen failed for site not sending \
                           Connection:close")
            else:
                self.assertTrue(res)

            req = urllib.request.urlopen(URL)
            res = req.read()
199 200
            self.assertTrue(res)

Georg Brandl's avatar
Georg Brandl committed
201
    def _test_urls(self, urls, handlers, retry=True):
202 203 204 205
        import time
        import logging
        debug = logging.getLogger("test_urllib2").debug

206
        urlopen = urllib.request.build_opener(*handlers).open
Georg Brandl's avatar
Georg Brandl committed
207
        if retry:
208
            urlopen = _wrap_with_retry_thrice(urlopen, urllib.error.URLError)
209 210

        for url in urls:
211 212 213
            with self.subTest(url=url):
                if isinstance(url, tuple):
                    url, req, expected_err = url
214
                else:
215 216 217
                    req = expected_err = None

                with support.transient_internet(url):
218
                    try:
219
                        f = urlopen(url, req, TIMEOUT)
220
                    # urllib.error.URLError is a subclass of OSError
221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
                    except OSError as err:
                        if expected_err:
                            msg = ("Didn't get expected error(s) %s for %s %s, got %s: %s" %
                                   (expected_err, url, req, type(err), err))
                            self.assertIsInstance(err, expected_err, msg)
                        else:
                            raise
                    else:
                        try:
                            with support.time_out, \
                                 support.socket_peer_reset, \
                                 support.ioerror_peer_reset:
                                buf = f.read()
                                debug("read %d bytes" % len(buf))
                        except socket.timeout:
                            print("<timeout: %s>" % url, file=sys.stderr)
                        f.close()
                time.sleep(0.1)
239 240 241 242

    def _extra_handlers(self):
        handlers = []

243
        cfh = urllib.request.CacheFTPHandler()
244
        self.addCleanup(cfh.clear_cache)
245 246 247 248 249
        cfh.setTimeout(1)
        handlers.append(cfh)

        return handlers

250

251 252
class TimeoutTest(unittest.TestCase):
    def test_http_basic(self):
253
        self.assertIsNone(socket.getdefaulttimeout())
254
        url = "http://www.example.com"
255 256
        with support.transient_internet(url, timeout=None):
            u = _urlopen_with_retry(url)
257
            self.addCleanup(u.close)
258
            self.assertIsNone(u.fp.raw._sock.gettimeout())
259

Georg Brandl's avatar
Georg Brandl committed
260
    def test_http_default_timeout(self):
261
        self.assertIsNone(socket.getdefaulttimeout())
262
        url = "http://www.example.com"
263 264 265 266
        with support.transient_internet(url):
            socket.setdefaulttimeout(60)
            try:
                u = _urlopen_with_retry(url)
267
                self.addCleanup(u.close)
268 269 270
            finally:
                socket.setdefaulttimeout(None)
            self.assertEqual(u.fp.raw._sock.gettimeout(), 60)
Georg Brandl's avatar
Georg Brandl committed
271 272

    def test_http_no_timeout(self):
273
        self.assertIsNone(socket.getdefaulttimeout())
274
        url = "http://www.example.com"
275 276 277 278
        with support.transient_internet(url):
            socket.setdefaulttimeout(60)
            try:
                u = _urlopen_with_retry(url, timeout=None)
279
                self.addCleanup(u.close)
280 281
            finally:
                socket.setdefaulttimeout(None)
282
            self.assertIsNone(u.fp.raw._sock.gettimeout())
283

Georg Brandl's avatar
Georg Brandl committed
284
    def test_http_timeout(self):
285
        url = "http://www.example.com"
286 287
        with support.transient_internet(url):
            u = _urlopen_with_retry(url, timeout=120)
288
            self.addCleanup(u.close)
289
            self.assertEqual(u.fp.raw._sock.gettimeout(), 120)
290

291
    FTP_HOST = 'ftp://www.pythontest.net/'
292

293
    def test_ftp_basic(self):
294
        self.assertIsNone(socket.getdefaulttimeout())
295 296
        with support.transient_internet(self.FTP_HOST, timeout=None):
            u = _urlopen_with_retry(self.FTP_HOST)
297
            self.addCleanup(u.close)
298
            self.assertIsNone(u.fp.fp.raw._sock.gettimeout())
299

Georg Brandl's avatar
Georg Brandl committed
300
    def test_ftp_default_timeout(self):
301
        self.assertIsNone(socket.getdefaulttimeout())
302 303 304 305
        with support.transient_internet(self.FTP_HOST):
            socket.setdefaulttimeout(60)
            try:
                u = _urlopen_with_retry(self.FTP_HOST)
306
                self.addCleanup(u.close)
307 308 309
            finally:
                socket.setdefaulttimeout(None)
            self.assertEqual(u.fp.fp.raw._sock.gettimeout(), 60)
310

Georg Brandl's avatar
Georg Brandl committed
311
    def test_ftp_no_timeout(self):
312
        self.assertIsNone(socket.getdefaulttimeout())
313 314 315 316
        with support.transient_internet(self.FTP_HOST):
            socket.setdefaulttimeout(60)
            try:
                u = _urlopen_with_retry(self.FTP_HOST, timeout=None)
317
                self.addCleanup(u.close)
318 319
            finally:
                socket.setdefaulttimeout(None)
320
            self.assertIsNone(u.fp.fp.raw._sock.gettimeout())
321

Georg Brandl's avatar
Georg Brandl committed
322
    def test_ftp_timeout(self):
323 324
        with support.transient_internet(self.FTP_HOST):
            u = _urlopen_with_retry(self.FTP_HOST, timeout=60)
325
            self.addCleanup(u.close)
326
            self.assertEqual(u.fp.fp.raw._sock.gettimeout(), 60)
327

328

329
if __name__ == "__main__":
330
    unittest.main()