test_sys.py 39.1 KB
Newer Older
1
import unittest, test.support
2
from test.script_helper import assert_python_ok, assert_python_failure
3
import sys, io, os
4
import struct
5 6
import subprocess
import textwrap
7
import warnings
8
import operator
9
import codecs
10
import gc
11
import sysconfig
12
import platform
13

14 15 16 17
# count the number of test runs, used to create unique
# strings to intern in test_intern()
numruns = 0

18 19 20 21
try:
    import threading
except ImportError:
    threading = None
22

23 24
class SysModuleTest(unittest.TestCase):

25 26 27 28 29 30 31 32 33
    def setUp(self):
        self.orig_stdout = sys.stdout
        self.orig_stderr = sys.stderr
        self.orig_displayhook = sys.displayhook

    def tearDown(self):
        sys.stdout = self.orig_stdout
        sys.stderr = self.orig_stderr
        sys.displayhook = self.orig_displayhook
34
        test.support.reap_children()
35

36
    def test_original_displayhook(self):
37
        import builtins
38
        out = io.StringIO()
39 40 41 42 43
        sys.stdout = out

        dh = sys.__displayhook__

        self.assertRaises(TypeError, dh)
44 45
        if hasattr(builtins, "_"):
            del builtins._
46 47 48

        dh(None)
        self.assertEqual(out.getvalue(), "")
49
        self.assertTrue(not hasattr(builtins, "_"))
50 51
        dh(42)
        self.assertEqual(out.getvalue(), "42\n")
52
        self.assertEqual(builtins._, 42)
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69

        del sys.stdout
        self.assertRaises(RuntimeError, dh, 42)

    def test_lost_displayhook(self):
        del sys.displayhook
        code = compile("42", "<string>", "single")
        self.assertRaises(RuntimeError, eval, code)

    def test_custom_displayhook(self):
        def baddisplayhook(obj):
            raise ValueError
        sys.displayhook = baddisplayhook
        code = compile("42", "<string>", "single")
        self.assertRaises(ValueError, eval, code)

    def test_original_excepthook(self):
70
        err = io.StringIO()
71 72 73 74 75 76 77
        sys.stderr = err

        eh = sys.__excepthook__

        self.assertRaises(TypeError, eh)
        try:
            raise ValueError(42)
78
        except ValueError as exc:
79 80
            eh(*sys.exc_info())

81
        self.assertTrue(err.getvalue().endswith("ValueError: 42\n"))
82

83 84 85
    def test_excepthook(self):
        with test.support.captured_output("stderr") as stderr:
            sys.excepthook(1, '1', 1)
86
        self.assertTrue("TypeError: print_exception(): Exception expected for " \
87 88
                         "value, str found" in stderr.getvalue())

Walter Dörwald's avatar
Walter Dörwald committed
89
    # FIXME: testing the code for a lost or replaced excepthook in
90 91 92
    # Python/pythonrun.c::PyErr_PrintEx() is tricky.

    def test_exit(self):
93
        # call with two arguments
94 95 96
        self.assertRaises(TypeError, sys.exit, 42, 42)

        # call without argument
97 98 99 100
        with self.assertRaises(SystemExit) as cm:
            sys.exit()
        self.assertIsNone(cm.exception.code)

101 102 103 104
        rc, out, err = assert_python_ok('-c', 'import sys; sys.exit()')
        self.assertEqual(rc, 0)
        self.assertEqual(out, b'')
        self.assertEqual(err, b'')
105

106 107
        # call with integer argument
        with self.assertRaises(SystemExit) as cm:
108
            sys.exit(42)
109
        self.assertEqual(cm.exception.code, 42)
110

111 112 113
        # call with tuple argument with one entry
        # entry will be unpacked
        with self.assertRaises(SystemExit) as cm:
114
            sys.exit((42,))
115
        self.assertEqual(cm.exception.code, 42)
116 117

        # call with string argument
118
        with self.assertRaises(SystemExit) as cm:
119
            sys.exit("exit")
120
        self.assertEqual(cm.exception.code, "exit")
121 122

        # call with tuple argument with two entries
123
        with self.assertRaises(SystemExit) as cm:
124
            sys.exit((17, 23))
125
        self.assertEqual(cm.exception.code, (17, 23))
126

127
        # test that the exit machinery handles SystemExits properly
128
        rc, out, err = assert_python_failure('-c', 'raise SystemExit(47)')
129
        self.assertEqual(rc, 47)
130 131
        self.assertEqual(out, b'')
        self.assertEqual(err, b'')
Tim Peters's avatar
Tim Peters committed
132

133 134 135 136 137 138
        def check_exit_message(code, expected, **env_vars):
            rc, out, err = assert_python_failure('-c', code, **env_vars)
            self.assertEqual(rc, 1)
            self.assertEqual(out, b'')
            self.assertTrue(err.startswith(expected),
                "%s doesn't start with %s" % (ascii(err), ascii(expected)))
139

140
        # test that stderr buffer is flushed before the exit message is written
141 142 143 144 145
        # into stderr
        check_exit_message(
            r'import sys; sys.stderr.write("unflushed,"); sys.exit("message")',
            b"unflushed,message")

146 147
        # test that the exit message is written with backslashreplace error
        # handler to stderr
148 149 150
        check_exit_message(
            r'import sys; sys.exit("surrogates:\uDCFF")',
            b"surrogates:\\udcff")
151

152 153 154 155
        # test that the unicode message is encoded to the stderr encoding
        # instead of the default encoding (utf8)
        check_exit_message(
            r'import sys; sys.exit("h\xe9")',
156
            b"h\xe9", PYTHONIOENCODING='latin-1')
157

158
    def test_getdefaultencoding(self):
159 160
        self.assertRaises(TypeError, sys.getdefaultencoding, 42)
        # can't check more than the type, as the user might have changed it
161
        self.assertIsInstance(sys.getdefaultencoding(), str)
162

163 164
    # testing sys.settrace() is done in test_sys_settrace.py
    # testing sys.setprofile() is done in test_sys_setprofile.py
165 166

    def test_setcheckinterval(self):
167 168 169 170 171 172
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            self.assertRaises(TypeError, sys.setcheckinterval)
            orig = sys.getcheckinterval()
            for n in 0, 100, 120, orig: # orig last to restore starting state
                sys.setcheckinterval(n)
173
                self.assertEqual(sys.getcheckinterval(), n)
174

175
    @unittest.skipUnless(threading, 'Threading required for this test.')
Antoine Pitrou's avatar
Antoine Pitrou committed
176 177 178 179 180 181 182 183 184 185 186
    def test_switchinterval(self):
        self.assertRaises(TypeError, sys.setswitchinterval)
        self.assertRaises(TypeError, sys.setswitchinterval, "a")
        self.assertRaises(ValueError, sys.setswitchinterval, -1.0)
        self.assertRaises(ValueError, sys.setswitchinterval, 0.0)
        orig = sys.getswitchinterval()
        # sanity check
        self.assertTrue(orig < 0.5, orig)
        try:
            for n in 0.00001, 0.05, 3.0, orig:
                sys.setswitchinterval(n)
187
                self.assertAlmostEqual(sys.getswitchinterval(), n)
Antoine Pitrou's avatar
Antoine Pitrou committed
188 189 190
        finally:
            sys.setswitchinterval(orig)

191
    def test_recursionlimit(self):
Tim Peters's avatar
Tim Peters committed
192 193 194 195 196 197 198
        self.assertRaises(TypeError, sys.getrecursionlimit, 42)
        oldlimit = sys.getrecursionlimit()
        self.assertRaises(TypeError, sys.setrecursionlimit)
        self.assertRaises(ValueError, sys.setrecursionlimit, -42)
        sys.setrecursionlimit(10000)
        self.assertEqual(sys.getrecursionlimit(), 10000)
        sys.setrecursionlimit(oldlimit)
199

200 201
    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
                     'fatal error if run with a trace function')
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232
    def test_recursionlimit_recovery(self):
        # NOTE: this test is slightly fragile in that it depends on the current
        # recursion count when executing the test being low enough so as to
        # trigger the recursion recovery detection in the _Py_MakeEndRecCheck
        # macro (see ceval.h).
        oldlimit = sys.getrecursionlimit()
        def f():
            f()
        try:
            for i in (50, 1000):
                # Issue #5392: stack overflow after hitting recursion limit twice
                sys.setrecursionlimit(i)
                self.assertRaises(RuntimeError, f)
                self.assertRaises(RuntimeError, f)
        finally:
            sys.setrecursionlimit(oldlimit)

    def test_recursionlimit_fatalerror(self):
        # A fatal error occurs if a second recursion limit is hit when recovering
        # from a first one.
        code = textwrap.dedent("""
            import sys

            def f():
                try:
                    f()
                except RuntimeError:
                    f()

            sys.setrecursionlimit(%d)
            f()""")
233
        with test.support.SuppressCrashReport():
234 235 236 237 238 239 240 241
            for i in (50, 1000):
                sub = subprocess.Popen([sys.executable, '-c', code % i],
                    stderr=subprocess.PIPE)
                err = sub.communicate()[1]
                self.assertTrue(sub.returncode, sub.returncode)
                self.assertIn(
                    b"Fatal Python error: Cannot recover from stack overflow",
                    err)
242

243
    def test_getwindowsversion(self):
244
        # Raise SkipTest if sys doesn't have getwindowsversion attribute
245
        test.support.get_attribute(sys, "getwindowsversion")
246
        v = sys.getwindowsversion()
Brian Curtin's avatar
Brian Curtin committed
247
        self.assertEqual(len(v), 5)
248 249 250 251 252 253 254 255 256 257 258
        self.assertIsInstance(v[0], int)
        self.assertIsInstance(v[1], int)
        self.assertIsInstance(v[2], int)
        self.assertIsInstance(v[3], int)
        self.assertIsInstance(v[4], str)
        self.assertRaises(IndexError, operator.getitem, v, 5)
        self.assertIsInstance(v.major, int)
        self.assertIsInstance(v.minor, int)
        self.assertIsInstance(v.build, int)
        self.assertIsInstance(v.platform, int)
        self.assertIsInstance(v.service_pack, str)
259 260 261 262
        self.assertIsInstance(v.service_pack_minor, int)
        self.assertIsInstance(v.service_pack_major, int)
        self.assertIsInstance(v.suite_mask, int)
        self.assertIsInstance(v.product_type, int)
263 264 265 266 267 268 269 270 271
        self.assertEqual(v[0], v.major)
        self.assertEqual(v[1], v.minor)
        self.assertEqual(v[2], v.build)
        self.assertEqual(v[3], v.platform)
        self.assertEqual(v[4], v.service_pack)

        # This is how platform.py calls it. Make sure tuple
        #  still has 5 elements
        maj, min, buildno, plat, csd = sys.getwindowsversion()
272

273 274 275
    def test_call_tracing(self):
        self.assertRaises(TypeError, sys.call_tracing, type, 2)

276 277
    @unittest.skipUnless(hasattr(sys, "setdlopenflags"),
                         'test needs sys.setdlopenflags()')
278
    def test_dlopenflags(self):
279 280 281 282 283 284 285
        self.assertTrue(hasattr(sys, "getdlopenflags"))
        self.assertRaises(TypeError, sys.getdlopenflags, 42)
        oldflags = sys.getdlopenflags()
        self.assertRaises(TypeError, sys.setdlopenflags)
        sys.setdlopenflags(oldflags+1)
        self.assertEqual(sys.getdlopenflags(), oldflags+1)
        sys.setdlopenflags(oldflags)
286

287
    @test.support.refcount_test
288
    def test_refcount(self):
Benjamin Peterson's avatar
Benjamin Peterson committed
289 290 291 292 293
        # n here must be a global in order for this test to pass while
        # tracing with a python function.  Tracing calls PyFrame_FastToLocals
        # which will add a copy of any locals to the frame object, causing
        # the reference count to increase by 2 instead of 1.
        global n
294 295 296 297 298 299 300
        self.assertRaises(TypeError, sys.getrefcount)
        c = sys.getrefcount(None)
        n = None
        self.assertEqual(sys.getrefcount(None), c+1)
        del n
        self.assertEqual(sys.getrefcount(None), c)
        if hasattr(sys, "gettotalrefcount"):
301
            self.assertIsInstance(sys.gettotalrefcount(), int)
302 303 304

    def test_getframe(self):
        self.assertRaises(TypeError, sys._getframe, 42, 42)
305
        self.assertRaises(ValueError, sys._getframe, 2000000000)
306
        self.assertTrue(
307
            SysModuleTest.test_getframe.__code__ \
308 309 310
            is sys._getframe().f_code
        )

311 312 313 314
    # sys._current_frames() is a CPython-only gimmick.
    def test_current_frames(self):
        have_threads = True
        try:
315
            import _thread
316 317 318 319 320 321 322 323 324
        except ImportError:
            have_threads = False

        if have_threads:
            self.current_frames_with_threads()
        else:
            self.current_frames_without_threads()

    # Test sys._current_frames() in a WITH_THREADS build.
325
    @test.support.reap_threads
326
    def current_frames_with_threads(self):
327
        import threading
328 329 330 331 332 333 334 335 336 337 338 339 340
        import traceback

        # Spawn a thread that blocks at a known place.  Then the main
        # thread does sys._current_frames(), and verifies that the frames
        # returned make sense.
        entered_g = threading.Event()
        leave_g = threading.Event()
        thread_info = []  # the thread's id

        def f123():
            g456()

        def g456():
341
            thread_info.append(threading.get_ident())
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356
            entered_g.set()
            leave_g.wait()

        t = threading.Thread(target=f123)
        t.start()
        entered_g.wait()

        # At this point, t has finished its entered_g.set(), although it's
        # impossible to guess whether it's still on that line or has moved on
        # to its leave_g.wait().
        self.assertEqual(len(thread_info), 1)
        thread_id = thread_info[0]

        d = sys._current_frames()

357
        main_id = threading.get_ident()
358 359
        self.assertIn(main_id, d)
        self.assertIn(thread_id, d)
360 361 362

        # Verify that the captured main-thread frame is _this_ frame.
        frame = d.pop(main_id)
363
        self.assertTrue(frame is sys._getframe())
364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380

        # Verify that the captured thread frame is blocked in g456, called
        # from f123.  This is a litte tricky, since various bits of
        # threading.py are also in the thread's call stack.
        frame = d.pop(thread_id)
        stack = traceback.extract_stack(frame)
        for i, (filename, lineno, funcname, sourceline) in enumerate(stack):
            if funcname == "f123":
                break
        else:
            self.fail("didn't find f123() on thread's call stack")

        self.assertEqual(sourceline, "g456()")

        # And the next record must be for g456().
        filename, lineno, funcname, sourceline = stack[i+1]
        self.assertEqual(funcname, "g456")
381
        self.assertIn(sourceline, ["leave_g.wait()", "entered_g.set()"])
382 383 384 385 386 387 388 389 390 391 392

        # Reap the spawned thread.
        leave_g.set()
        t.join()

    # Test sys._current_frames() when thread support doesn't exist.
    def current_frames_without_threads(self):
        # Not much happens here:  there is only one thread, with artificial
        # "thread id" 0.
        d = sys._current_frames()
        self.assertEqual(len(d), 1)
393
        self.assertIn(0, d)
394
        self.assertTrue(d[0] is sys._getframe())
395

396
    def test_attributes(self):
397 398
        self.assertIsInstance(sys.api_version, int)
        self.assertIsInstance(sys.argv, list)
399
        self.assertIn(sys.byteorder, ("little", "big"))
400 401 402
        self.assertIsInstance(sys.builtin_module_names, tuple)
        self.assertIsInstance(sys.copyright, str)
        self.assertIsInstance(sys.exec_prefix, str)
403
        self.assertIsInstance(sys.base_exec_prefix, str)
404
        self.assertIsInstance(sys.executable, str)
405
        self.assertEqual(len(sys.float_info), 11)
406
        self.assertEqual(sys.float_info.radix, 2)
407
        self.assertEqual(len(sys.int_info), 2)
408 409
        self.assertTrue(sys.int_info.bits_per_digit % 5 == 0)
        self.assertTrue(sys.int_info.sizeof_digit >= 1)
Benjamin Peterson's avatar
Benjamin Peterson committed
410 411
        self.assertEqual(type(sys.int_info.bits_per_digit), int)
        self.assertEqual(type(sys.int_info.sizeof_digit), int)
412
        self.assertIsInstance(sys.hexversion, int)
413

414
        self.assertEqual(len(sys.hash_info), 9)
415 416 417 418 419 420 421 422 423 424 425 426 427 428
        self.assertLess(sys.hash_info.modulus, 2**sys.hash_info.width)
        # sys.hash_info.modulus should be a prime; we do a quick
        # probable primality test (doesn't exclude the possibility of
        # a Carmichael number)
        for x in range(1, 100):
            self.assertEqual(
                pow(x, sys.hash_info.modulus-1, sys.hash_info.modulus),
                1,
                "sys.hash_info.modulus {} is a non-prime".format(
                    sys.hash_info.modulus)
                )
        self.assertIsInstance(sys.hash_info.inf, int)
        self.assertIsInstance(sys.hash_info.nan, int)
        self.assertIsInstance(sys.hash_info.imag, int)
429
        algo = sysconfig.get_config_var("Py_HASH_ALGORITHM")
430 431 432 433 434 435 436 437 438
        if sys.hash_info.algorithm in {"fnv", "siphash24"}:
            self.assertIn(sys.hash_info.hash_bits, {32, 64})
            self.assertIn(sys.hash_info.seed_bits, {32, 64, 128})

            if algo == 1:
                self.assertEqual(sys.hash_info.algorithm, "siphash24")
            elif algo == 2:
                self.assertEqual(sys.hash_info.algorithm, "fnv")
            else:
439
                self.assertIn(sys.hash_info.algorithm, {"fnv", "siphash24"})
440 441 442 443 444
        else:
            # PY_HASH_EXTERNAL
            self.assertEqual(algo, 0)
        self.assertGreaterEqual(sys.hash_info.cutoff, 0)
        self.assertLess(sys.hash_info.cutoff, 8)
445

446 447
        self.assertIsInstance(sys.maxsize, int)
        self.assertIsInstance(sys.maxunicode, int)
448
        self.assertEqual(sys.maxunicode, 0x10FFFF)
449 450
        self.assertIsInstance(sys.platform, str)
        self.assertIsInstance(sys.prefix, str)
451
        self.assertIsInstance(sys.base_prefix, str)
452
        self.assertIsInstance(sys.version, str)
453
        vi = sys.version_info
454
        self.assertIsInstance(vi[:], tuple)
455
        self.assertEqual(len(vi), 5)
456 457 458
        self.assertIsInstance(vi[0], int)
        self.assertIsInstance(vi[1], int)
        self.assertIsInstance(vi[2], int)
459
        self.assertIn(vi[3], ("alpha", "beta", "candidate", "final"))
460 461 462 463
        self.assertIsInstance(vi[4], int)
        self.assertIsInstance(vi.major, int)
        self.assertIsInstance(vi.minor, int)
        self.assertIsInstance(vi.micro, int)
464
        self.assertIn(vi.releaselevel, ("alpha", "beta", "candidate", "final"))
465
        self.assertIsInstance(vi.serial, int)
466 467 468 469 470
        self.assertEqual(vi[0], vi.major)
        self.assertEqual(vi[1], vi.minor)
        self.assertEqual(vi[2], vi.micro)
        self.assertEqual(vi[3], vi.releaselevel)
        self.assertEqual(vi[4], vi.serial)
471
        self.assertTrue(vi > (1,0,0))
472
        self.assertIsInstance(sys.float_repr_style, str)
473
        self.assertIn(sys.float_repr_style, ('short', 'legacy'))
474 475
        if not sys.platform.startswith('win'):
            self.assertIsInstance(sys.abiflags, str)
476

477 478 479 480
    @unittest.skipUnless(hasattr(sys, 'thread_info'),
                         'Threading required for this test.')
    def test_thread_info(self):
        info = sys.thread_info
Ezio Melotti's avatar
Ezio Melotti committed
481
        self.assertEqual(len(info), 3)
482
        self.assertIn(info.name, ('nt', 'pthread', 'solaris', None))
483 484
        self.assertIn(info.lock, ('semaphore', 'mutex+cond', None))

485
    def test_43581(self):
486
        # Can't use sys.stdout, as this is a StringIO object when
487
        # the test runs under regrtest.
488
        self.assertEqual(sys.__stdout__.encoding, sys.__stderr__.encoding)
489

490
    def test_intern(self):
491 492
        global numruns
        numruns += 1
493
        self.assertRaises(TypeError, sys.intern)
494
        s = "never interned before" + str(numruns)
495
        self.assertTrue(sys.intern(s) is s)
496
        s2 = s.swapcase().swapcase()
497
        self.assertTrue(sys.intern(s2) is s)
498 499 500 501 502

        # Subclasses of string can't be interned, because they
        # provide too much opportunity for insane things to happen.
        # We don't want them in the interned dict and if they aren't
        # actually interned, we don't want to create the appearance
503
        # that they are by allowing intern() to succeed.
504
        class S(str):
505 506 507
            def __hash__(self):
                return 123

508
        self.assertRaises(TypeError, sys.intern, S("abc"))
509

510
    def test_sys_flags(self):
511
        self.assertTrue(sys.flags)
512
        attrs = ("debug",
513
                 "inspect", "interactive", "optimize", "dont_write_bytecode",
514
                 "no_user_site", "no_site", "ignore_environment", "verbose",
515
                 "bytes_warning", "quiet", "hash_randomization", "isolated")
516
        for attr in attrs:
517
            self.assertTrue(hasattr(sys.flags, attr), attr)
518
            self.assertEqual(type(getattr(sys.flags, attr)), int, attr)
519
        self.assertTrue(repr(sys.flags))
520
        self.assertEqual(len(sys.flags), len(attrs))
521

522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541
    def assert_raise_on_new_sys_type(self, sys_attr):
        # Users are intentionally prevented from creating new instances of
        # sys.flags, sys.version_info, and sys.getwindowsversion.
        attr_type = type(sys_attr)
        with self.assertRaises(TypeError):
            attr_type()
        with self.assertRaises(TypeError):
            attr_type.__new__(attr_type)

    def test_sys_flags_no_instantiation(self):
        self.assert_raise_on_new_sys_type(sys.flags)

    def test_sys_version_info_no_instantiation(self):
        self.assert_raise_on_new_sys_type(sys.version_info)

    def test_sys_getwindowsversion_no_instantiation(self):
        # Skip if not being run on Windows.
        test.support.get_attribute(sys, "getwindowsversion")
        self.assert_raise_on_new_sys_type(sys.getwindowsversion())

Christian Heimes's avatar
Christian Heimes committed
542 543 544
    def test_clear_type_cache(self):
        sys._clear_type_cache()

545 546 547 548 549 550 551 552 553
    def test_ioencoding(self):
        env = dict(os.environ)

        # Test character: cent sign, encoded as 0x4A (ASCII J) in CP424,
        # not representable in ASCII.

        env["PYTHONIOENCODING"] = "cp424"
        p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xa2))'],
                             stdout = subprocess.PIPE, env=env)
554
        out = p.communicate()[0].strip()
555 556
        expected = ("\xa2" + os.linesep).encode("cp424")
        self.assertEqual(out, expected)
557 558 559 560

        env["PYTHONIOENCODING"] = "ascii:replace"
        p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xa2))'],
                             stdout = subprocess.PIPE, env=env)
561
        out = p.communicate()[0].strip()
562 563
        self.assertEqual(out, b'?')

564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599
        env["PYTHONIOENCODING"] = "ascii"
        p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xa2))'],
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                             env=env)
        out, err = p.communicate()
        self.assertEqual(out, b'')
        self.assertIn(b'UnicodeEncodeError:', err)
        self.assertIn(rb"'\xa2'", err)

        env["PYTHONIOENCODING"] = "ascii:"
        p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xa2))'],
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                             env=env)
        out, err = p.communicate()
        self.assertEqual(out, b'')
        self.assertIn(b'UnicodeEncodeError:', err)
        self.assertIn(rb"'\xa2'", err)

        env["PYTHONIOENCODING"] = ":surrogateescape"
        p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xdcbd))'],
                             stdout=subprocess.PIPE, env=env)
        out = p.communicate()[0].strip()
        self.assertEqual(out, b'\xbd')

    @unittest.skipUnless(test.support.FS_NONASCII,
                         'requires OS support of non-ASCII encodings')
    def test_ioencoding_nonascii(self):
        env = dict(os.environ)

        env["PYTHONIOENCODING"] = ""
        p = subprocess.Popen([sys.executable, "-c",
                                'print(%a)' % test.support.FS_NONASCII],
                                stdout=subprocess.PIPE, env=env)
        out = p.communicate()[0].strip()
        self.assertEqual(out, os.fsencode(test.support.FS_NONASCII))

600 601
    @unittest.skipIf(sys.base_prefix != sys.prefix,
                     'Test is not venv-compatible')
602
    def test_executable(self):
603 604 605
        # sys.executable should be absolute
        self.assertEqual(os.path.abspath(sys.executable), sys.executable)

606 607 608
        # Issue #7774: Ensure that sys.executable is an empty string if argv[0]
        # has been set to an non existent program name and Python is unable to
        # retrieve the real program name
Florent Xicluna's avatar
Florent Xicluna committed
609

610 611 612 613 614 615 616 617 618 619 620 621
        # For a normal installation, it should work without 'cwd'
        # argument. For test runs in the build directory, see #7774.
        python_dir = os.path.dirname(os.path.realpath(sys.executable))
        p = subprocess.Popen(
            ["nonexistent", "-c",
             'import sys; print(sys.executable.encode("ascii", "backslashreplace"))'],
            executable=sys.executable, stdout=subprocess.PIPE, cwd=python_dir)
        stdout = p.communicate()[0]
        executable = stdout.strip().decode("ASCII")
        p.wait()
        self.assertIn(executable, ["b''", repr(sys.executable.encode("ascii", "backslashreplace"))])

622 623 624 625 626
    def check_fsencoding(self, fs_encoding, expected=None):
        self.assertIsNotNone(fs_encoding)
        codecs.lookup(fs_encoding)
        if expected:
            self.assertEqual(fs_encoding, expected)
627

628
    def test_getfilesystemencoding(self):
629
        fs_encoding = sys.getfilesystemencoding()
630 631 632 633 634 635
        if sys.platform == 'darwin':
            expected = 'utf-8'
        elif sys.platform == 'win32':
            expected = 'mbcs'
        else:
            expected = None
636
        self.check_fsencoding(fs_encoding, expected)
637

638
    def c_locale_get_error_handler(self, isolated=False, encoding=None):
639 640 641 642
        # Force the POSIX locale
        env = os.environ.copy()
        env["LC_ALL"] = "C"
        code = '\n'.join((
643
            'import sys',
644 645
            'def dump(name):',
            '    std = getattr(sys, name)',
646
            '    print("%s: %s" % (name, std.errors))',
647 648 649 650
            'dump("stdin")',
            'dump("stdout")',
            'dump("stderr")',
        ))
651 652 653 654 655 656 657 658 659 660 661 662 663 664 665
        args = [sys.executable, "-c", code]
        if isolated:
            args.append("-I")
        elif encoding:
            env['PYTHONIOENCODING'] = encoding
        p = subprocess.Popen(args,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.STDOUT,
                              env=env,
                              universal_newlines=True)
        stdout, stderr = p.communicate()
        return stdout

    def test_c_locale_surrogateescape(self):
        out = self.c_locale_get_error_handler(isolated=True)
666
        self.assertEqual(out,
667 668 669
                         'stdin: surrogateescape\n'
                         'stdout: surrogateescape\n'
                         'stderr: backslashreplace\n')
670 671

        # replace the default error handler
672
        out = self.c_locale_get_error_handler(encoding=':strict')
673
        self.assertEqual(out,
674 675 676
                         'stdin: strict\n'
                         'stdout: strict\n'
                         'stderr: backslashreplace\n')
677 678

        # force the encoding
679
        out = self.c_locale_get_error_handler(encoding='iso8859-1')
680
        self.assertEqual(out,
681 682 683
                         'stdin: surrogateescape\n'
                         'stdout: surrogateescape\n'
                         'stderr: backslashreplace\n')
684

685 686 687
    def test_implementation(self):
        # This test applies to all implementations equally.

688
        levels = {'alpha': 0xA, 'beta': 0xB, 'candidate': 0xC, 'final': 0xF}
689 690 691 692 693 694 695 696 697 698 699 700 701 702

        self.assertTrue(hasattr(sys.implementation, 'name'))
        self.assertTrue(hasattr(sys.implementation, 'version'))
        self.assertTrue(hasattr(sys.implementation, 'hexversion'))
        self.assertTrue(hasattr(sys.implementation, 'cache_tag'))

        version = sys.implementation.version
        self.assertEqual(version[:2], (version.major, version.minor))

        hexversion = (version.major << 24 | version.minor << 16 |
                      version.micro << 8 | levels[version.releaselevel] << 4 |
                      version.serial << 0)
        self.assertEqual(sys.implementation.hexversion, hexversion)

703
        # PEP 421 requires that .name be lower case.
Barry Warsaw's avatar
Barry Warsaw committed
704
        self.assertEqual(sys.implementation.name,
705 706
                         sys.implementation.name.lower())

707
    @test.support.cpython_only
708 709 710 711 712 713
    def test_debugmallocstats(self):
        # Test sys._debugmallocstats()
        from test.script_helper import assert_python_ok
        args = ['-c', 'import sys; sys._debugmallocstats()']
        ret, out, err = assert_python_ok(*args)
        self.assertIn(b"free PyDictObjects", err)
714

715 716 717
        # The function has no parameter
        self.assertRaises(TypeError, sys._debugmallocstats, True)

718 719 720 721
    @unittest.skipUnless(hasattr(sys, "getallocatedblocks"),
                         "sys.getallocatedblocks unavailable on this build")
    def test_getallocatedblocks(self):
        # Some sanity checks
722
        with_pymalloc = sysconfig.get_config_var('WITH_PYMALLOC')
723 724
        a = sys.getallocatedblocks()
        self.assertIs(type(a), int)
725 726 727
        if with_pymalloc:
            self.assertGreater(a, 0)
        else:
728 729 730 731
            # When WITH_PYMALLOC isn't available, we don't know anything
            # about the underlying implementation: the function might
            # return 0 or something greater.
            self.assertGreaterEqual(a, 0)
732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747
        try:
            # While we could imagine a Python session where the number of
            # multiple buffer objects would exceed the sharing of references,
            # it is unlikely to happen in a normal test run.
            self.assertLess(a, sys.gettotalrefcount())
        except AttributeError:
            # gettotalrefcount() not available
            pass
        gc.collect()
        b = sys.getallocatedblocks()
        self.assertLessEqual(b, a)
        gc.collect()
        c = sys.getallocatedblocks()
        self.assertIn(c, range(b - 50, b + 50))


748
@test.support.cpython_only
749 750 751
class SizeofTest(unittest.TestCase):

    def setUp(self):
752
        self.P = struct.calcsize('P')
753
        self.longdigit = sys.int_info.sizeof_digit
754 755
        import _testcapi
        self.gc_headsize = _testcapi.SIZEOF_PYGC_HEAD
756 757 758 759 760 761
        self.file = open(test.support.TESTFN, 'wb')

    def tearDown(self):
        self.file.close()
        test.support.unlink(test.support.TESTFN)

762
    check_sizeof = test.support.check_sizeof
763

764 765
    def test_gc_head_size(self):
        # Check that the gc header size is added to objects tracked by the gc.
766
        vsize = test.support.calcvobjsize
767 768
        gc_header_size = self.gc_headsize
        # bool objects are not gc tracked
769
        self.assertEqual(sys.getsizeof(True), vsize('') + self.longdigit)
770
        # but lists are
771
        self.assertEqual(sys.getsizeof([]), vsize('Pn') + gc_header_size)
772 773

    def test_default(self):
774 775 776
        size = test.support.calcvobjsize
        self.assertEqual(sys.getsizeof(True), size('') + self.longdigit)
        self.assertEqual(sys.getsizeof(True, -1), size('') + self.longdigit)
777 778 779

    def test_objecttypes(self):
        # check all types defined in Objects/
780 781
        size = test.support.calcobjsize
        vsize = test.support.calcvobjsize
782 783
        check = self.check_sizeof
        # bool
784
        check(True, vsize('') + self.longdigit)
785 786 787
        # buffer
        # XXX
        # builtin_function_or_method
788
        check(len, size('4P')) # XXX check layout
789 790 791 792
        # bytearray
        samples = [b'', b'u'*100000]
        for sample in samples:
            x = bytearray(sample)
793
            check(x, vsize('n2Pi') + x.__alloc__())
794
        # bytearray_iterator
795
        check(iter(bytearray()), size('nP'))
796 797 798 799 800 801
        # cell
        def get_cell():
            x = 42
            def inner():
                return x
            return inner
802
        check(get_cell().__closure__[0], size('P'))
803
        # code
804 805
        check(get_cell().__code__, size('5i9Pi3P'))
        check(get_cell.__code__, size('5i9Pi3P'))
806 807 808 809
        def get_cell2(x):
            def inner():
                return x
            return inner
810
        check(get_cell2.__code__, size('5i9Pi3P') + 1)
811
        # complex
812
        check(complex(0,1), size('2d'))
813
        # method_descriptor (descriptor object)
814
        check(str.lower, size('3PP'))
815 816 817 818
        # classmethod_descriptor (descriptor object)
        # XXX
        # member_descriptor (descriptor object)
        import datetime
819
        check(datetime.timedelta.days, size('3PP'))
820 821
        # getset_descriptor (descriptor object)
        import collections
822
        check(collections.defaultdict.default_factory, size('3PP'))
823
        # wrapper_descriptor (descriptor object)
824
        check(int.__add__, size('3P2P'))
825
        # method-wrapper (descriptor object)
826
        check({}.__iter__, size('2P'))
827
        # dict
828
        check({}, size('n2P' + '2nPn' + 8*'n2P'))
829
        longdict = {1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:8}
830
        check(longdict, size('n2P' + '2nPn') + 16*struct.calcsize('n2P'))
831
        # dictionary-keyiterator
832
        check({}.keys(), size('P'))
833
        # dictionary-valueiterator
834
        check({}.values(), size('P'))
835
        # dictionary-itemiterator
836 837
        check({}.items(), size('P'))
        # dictionary iterator
838
        check(iter({}), size('P2nPn'))
839 840
        # dictproxy
        class C(object): pass
841
        check(C.__dict__, size('P'))
842
        # BaseException
843
        check(BaseException(), size('5Pb'))
844
        # UnicodeEncodeError
845
        check(UnicodeEncodeError("", "", 0, 0, ""), size('5Pb 2P2nP'))
846
        # UnicodeDecodeError
847
        check(UnicodeDecodeError("", b"", 0, 0, ""), size('5Pb 2P2nP'))
848
        # UnicodeTranslateError
849
        check(UnicodeTranslateError("", 0, 1, ""), size('5Pb 2P2nP'))
850
        # ellipses
851
        check(Ellipsis, size(''))
852 853 854
        # EncodingMap
        import codecs, encodings.iso8859_3
        x = codecs.charmap_build(encodings.iso8859_3.decoding_table)
855
        check(x, size('32B2iB'))
856
        # enumerate
857
        check(enumerate([]), size('n3P'))
858
        # reverse
859
        check(reversed(''), size('nP'))
860
        # float
861
        check(float(0), size('d'))
862
        # sys.floatinfo
863
        check(sys.float_info, vsize('') + self.P * len(sys.float_info))
864 865 866 867 868 869 870 871
        # frame
        import inspect
        CO_MAXBLOCKS = 20
        x = inspect.currentframe()
        ncells = len(x.f_code.co_cellvars)
        nfrees = len(x.f_code.co_freevars)
        extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\
                  ncells + nfrees - 1
872
        check(x, vsize('12P3ic' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
873 874
        # function
        def func(): pass
875
        check(func, size('12P'))
876 877 878 879 880 881 882 883
        class c():
            @staticmethod
            def foo():
                pass
            @classmethod
            def bar(cls):
                pass
            # staticmethod
884
            check(foo, size('PP'))
885
            # classmethod
886
            check(bar, size('PP'))
887 888
        # generator
        def get_gen(): yield 1
889
        check(get_gen(), size('Pb2PPP'))
890
        # iterator
891
        check(iter('abc'), size('lP'))
892 893
        # callable-iterator
        import re
894
        check(re.finditer('',''), size('2P'))
895 896 897
        # list
        samples = [[], [1,2,3], ['1', '2', '3']]
        for sample in samples:
898
            check(sample, vsize('Pn') + len(sample)*self.P)
899 900 901 902 903
        # sortwrapper (list)
        # XXX
        # cmpwrapper (list)
        # XXX
        # listiterator (list)
904
        check(iter([]), size('lP'))
905
        # listreverseiterator (list)
906
        check(reversed([]), size('nP'))
907
        # int
908 909 910
        check(0, vsize(''))
        check(1, vsize('') + self.longdigit)
        check(-1, vsize('') + self.longdigit)
911
        PyLong_BASE = 2**sys.int_info.bits_per_digit
912 913 914
        check(int(PyLong_BASE), vsize('') + 2*self.longdigit)
        check(int(PyLong_BASE**2-1), vsize('') + 2*self.longdigit)
        check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit)
915
        # memoryview
916
        check(memoryview(b''), size('Pnin 2P2n2i5P 3cPn'))
917
        # module
918
        check(unittest, size('PnPPP'))
919
        # None
920
        check(None, size(''))
921
        # NotImplementedType
922
        check(NotImplemented, size(''))
923
        # object
924
        check(object(), size(''))
925 926 927 928 929 930
        # property (descriptor object)
        class C(object):
            def getx(self): return self.__x
            def setx(self, value): self.__x = value
            def delx(self): del self.__x
            x = property(getx, setx, delx, "")
931
            check(x, size('4Pi'))
932
        # PyCapsule
933 934
        # XXX
        # rangeiterator
935
        check(iter(range(1)), size('4l'))
936
        # reverse
937
        check(reversed(''), size('nP'))
938
        # range
939 940
        check(range(1), size('4P'))
        check(range(66000), size('4P'))
941 942 943 944
        # set
        # frozenset
        PySet_MINSIZE = 8
        samples = [[], range(10), range(50)]
945
        s = size('3n2P' + PySet_MINSIZE*'nP' + 'nP')
946 947 948 949 950 951 952 953 954 955 956 957 958
        for sample in samples:
            minused = len(sample)
            if minused == 0: tmp = 1
            # the computation of minused is actually a bit more complicated
            # but this suffices for the sizeof test
            minused = minused*2
            newsize = PySet_MINSIZE
            while newsize <= minused:
                newsize = newsize << 1
            if newsize <= 8:
                check(set(sample), s)
                check(frozenset(sample), s)
            else:
959 960
                check(set(sample), s + newsize*struct.calcsize('nP'))
                check(frozenset(sample), s + newsize*struct.calcsize('nP'))
961
        # setiterator
962
        check(iter(set()), size('P3n'))
963
        # slice
964
        check(slice(0), size('3P'))
965
        # super
966
        check(super(int), size('3P'))
967
        # tuple
968 969
        check((), vsize(''))
        check((1,2,3), vsize('') + 3*self.P)
970
        # type
971
        # static type: PyTypeObject
972
        s = vsize('P2n15Pl4Pn9Pn11PIP')
973
        check(int, s)
974 975
        # (PyTypeObject + PyNumberMethods + PyMappingMethods +
        #  PySequenceMethods + PyBufferProcs + 4P)
976
        s = vsize('P2n17Pl4Pn9Pn11PIP') + struct.calcsize('34P 3P 10P 2P 4P')
977
        # Separate block for PyDictKeysObject with 4 entries
978
        s += struct.calcsize("2nPn") + 4*struct.calcsize("n2P")
979 980 981
        # class
        class newstyleclass(object): pass
        check(newstyleclass, s)
982
        # dict with shared keys
983
        check(newstyleclass().__dict__, size('n2P' + '2nPn'))
Georg Brandl's avatar
Georg Brandl committed
984
        # unicode
Martin v. Löwis's avatar
Martin v. Löwis committed
985 986 987 988 989 990
        # each tuple contains a string and its expected character size
        # don't put any static strings here, as they may contain
        # wchar_t or UTF-8 representations
        samples = ['1'*100, '\xff'*50,
                   '\u0100'*40, '\uffff'*100,
                   '\U00010000'*30, '\U0010ffff'*100]
991
        asciifields = "nnbP"
992
        compactfields = asciifields + "nPn"
Martin v. Löwis's avatar
Martin v. Löwis committed
993
        unicodefields = compactfields + "P"
Georg Brandl's avatar
Georg Brandl committed
994
        for s in samples:
Martin v. Löwis's avatar
Martin v. Löwis committed
995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007
            maxchar = ord(max(s))
            if maxchar < 128:
                L = size(asciifields) + len(s) + 1
            elif maxchar < 256:
                L = size(compactfields) + len(s) + 1
            elif maxchar < 65536:
                L = size(compactfields) + 2*(len(s) + 1)
            else:
                L = size(compactfields) + 4*(len(s) + 1)
            check(s, L)
        # verify that the UTF-8 size is accounted for
        s = chr(0x4000)   # 4 bytes canonical representation
        check(s, size(compactfields) + 4)
1008 1009 1010 1011
        # compile() will trigger the generation of the UTF-8
        # representation as a side effect
        compile(s, "<stdin>", "eval")
        check(s, size(compactfields) + 4 + 4)
Martin v. Löwis's avatar
Martin v. Löwis committed
1012 1013
        # TODO: add check that forces the presence of wchar_t representation
        # TODO: add check that forces layout of unicodefields
1014 1015
        # weakref
        import weakref
1016
        check(weakref.ref(int), size('2Pn2P'))
1017 1018 1019
        # weakproxy
        # XXX
        # weakcallableproxy
1020
        check(weakref.proxy(int), size('2Pn2P'))
1021 1022 1023

    def test_pythontypes(self):
        # check all types defined in Python/
1024 1025
        size = test.support.calcobjsize
        vsize = test.support.calcvobjsize
1026 1027 1028
        check = self.check_sizeof
        # _ast.AST
        import _ast
1029
        check(_ast.AST(), size('P'))
1030 1031 1032 1033 1034
        try:
            raise TypeError
        except TypeError:
            tb = sys.exc_info()[2]
            # traceback
1035
            if tb is not None:
1036
                check(tb, size('2P2i'))
1037 1038 1039
        # symtable entry
        # XXX
        # sys.flags
1040
        check(sys.flags, vsize('') + self.P * len(sys.flags))
1041 1042


1043
def test_main():
1044
    test.support.run_unittest(SysModuleTest, SizeofTest)
1045 1046 1047

if __name__ == "__main__":
    test_main()