Issue #26040: Improve test_math and test_cmath coverage and rigour. Thanks Jeff Allen.

Lib/test/cmath_testcases.txt

@@ -53,6 +53,12 @@
 -- MPFR homepage at http://www.mpfr.org for more information about the
 -- MPFR project.
 
+-- A minority of the test cases were generated with the help of
+-- mpmath 0.19 at 100 bit accuracy (http://mpmath.org) to improve
+-- coverage of real functions with real-valued arguments. These are
+-- used in test.test_math.MathTests.test_testfile, as well as in
+-- test_cmath.
+
 
 --------------------------
 -- acos: Inverse cosine --
@@ -848,6 +854,18 @@ atan0302 atan 9.9999999999999999e-161 -1.0 -> 0.78539816339744828 -184.553381029
 atan0303 atan -1e-165 1.0 -> -0.78539816339744828 190.30984376228875
 atan0304 atan -9.9998886718268301e-321 -1.0 -> -0.78539816339744828 -368.76019403576692
 
+-- Additional real values (mpmath)
+atan0400 atan 1.7976931348623157e+308 0.0 -> 1.5707963267948966192 0.0
+atan0401 atan -1.7976931348623157e+308 0.0 -> -1.5707963267948966192 0.0
+atan0402 atan 1e-17 0.0 -> 1.0000000000000000715e-17 0.0
+atan0403 atan -1e-17 0.0 -> -1.0000000000000000715e-17 0.0
+atan0404 atan 0.0001 0.0 -> 0.000099999999666666673459 0.0
+atan0405 atan -0.0001 0.0 -> -0.000099999999666666673459 0.0
+atan0406 atan 0.999999999999999 0.0 -> 0.78539816339744781002 0.0
+atan0407 atan 1.000000000000001 0.0 -> 0.78539816339744886473 0.0
+atan0408 atan 14.101419947171719 0.0 -> 1.4999999999999999969 0.0
+atan0409 atan 1255.7655915007897 0.0 -> 1.5700000000000000622 0.0
+
 -- special values
 atan1000 atan -0.0 0.0 -> -0.0 0.0
 atan1001 atan nan 0.0 -> nan 0.0
@@ -1514,6 +1532,11 @@ sqrt0131 sqrt -1.5477066694467245e-310 -0.0 -> 0.0 -1.2440685951533077e-155
 sqrt0140 sqrt 1.6999999999999999e+308 -1.6999999999999999e+308 -> 1.4325088230154573e+154 -5.9336458271212207e+153
 sqrt0141 sqrt -1.797e+308 -9.9999999999999999e+306 -> 3.7284476432057307e+152 -1.3410406899802901e+154
 
+-- Additional real values (mpmath)
+sqrt0150 sqrt 1.7976931348623157e+308 0.0 -> 1.3407807929942596355e+154 0.0
+sqrt0151 sqrt 2.2250738585072014e-308 0.0 -> 1.4916681462400413487e-154 0.0
+sqrt0152 sqrt 5e-324 0.0 -> 2.2227587494850774834e-162 0.0
+
 -- special values
 sqrt1000 sqrt 0.0 0.0 -> 0.0 0.0
 sqrt1001 sqrt -0.0 0.0 -> 0.0 0.0
@@ -1616,6 +1639,20 @@ exp0052 exp 710.0 1.5 -> 1.5802653829857376e+307 inf    overflow
 exp0053 exp 710.0 1.6 -> -6.5231579995501372e+306 inf   overflow
 exp0054 exp 710.0 2.8 -> -inf 7.4836177417448528e+307   overflow
 
+-- Additional real values (mpmath)
+exp0070 exp 1e-08 0.0 -> 1.00000001000000005 0.0
+exp0071 exp 0.0003 0.0 -> 1.0003000450045003375 0.0
+exp0072 exp 0.2 0.0 -> 1.2214027581601698475 0.0
+exp0073 exp 1.0 0.0 -> 2.7182818284590452354 0.0
+exp0074 exp -1e-08 0.0 -> 0.99999999000000005 0.0
+exp0075 exp -0.0003 0.0 -> 0.99970004499550033751 0.0
+exp0076 exp -1.0 0.0 -> 0.3678794411714423216 0.0
+exp0077 exp 2.220446049250313e-16 0.0 -> 1.000000000000000222 0.0
+exp0078 exp -1.1102230246251565e-16 0.0 -> 0.99999999999999988898 0.0
+exp0079 exp 2.302585092994046 0.0 -> 10.000000000000002171 0.0
+exp0080 exp -2.302585092994046 0.0 -> 0.099999999999999978292 0.0
+exp0081 exp 709.7827 0.0 -> 1.7976699566638014654e+308 0.0
+
 -- special values
 exp1000 exp 0.0 0.0 -> 1.0 0.0
 exp1001 exp -0.0 0.0 -> 1.0 0.0
@@ -1708,6 +1745,23 @@ cosh0023 cosh 2.218885944363501 2.0015727395883687 -> -1.94294321081968 4.129026
 cosh0030 cosh 710.5 2.3519999999999999 -> -1.2967465239355998e+308 1.3076707908857333e+308
 cosh0031 cosh -710.5 0.69999999999999996 -> 1.4085466381392499e+308 -1.1864024666450239e+308
 
+-- Additional real values (mpmath)
+cosh0050 cosh 1e-150 0.0 -> 1.0 0.0
+cosh0051 cosh 1e-18 0.0 -> 1.0 0.0
+cosh0052 cosh 1e-09 0.0 -> 1.0000000000000000005 0.0
+cosh0053 cosh 0.0003 0.0 -> 1.0000000450000003375 0.0
+cosh0054 cosh 0.2 0.0 -> 1.0200667556190758485 0.0
+cosh0055 cosh 1.0 0.0 -> 1.5430806348152437785 0.0
+cosh0056 cosh -1e-18 0.0 -> 1.0 -0.0
+cosh0057 cosh -0.0003 0.0 -> 1.0000000450000003375 -0.0
+cosh0058 cosh -1.0 0.0 -> 1.5430806348152437785 -0.0
+cosh0059 cosh 1.3169578969248168 0.0 -> 2.0000000000000001504 0.0
+cosh0060 cosh -1.3169578969248168 0.0 -> 2.0000000000000001504 -0.0
+cosh0061 cosh 17.328679513998633 0.0 -> 16777216.000000021938 0.0
+cosh0062 cosh 18.714973875118524 0.0 -> 67108864.000000043662 0.0
+cosh0063 cosh 709.7827 0.0 -> 8.9883497833190073272e+307 0.0
+cosh0064 cosh -709.7827 0.0 -> 8.9883497833190073272e+307 -0.0
+
 -- special values
 cosh1000 cosh 0.0 0.0 -> 1.0 0.0
 cosh1001 cosh 0.0 inf -> nan 0.0        invalid ignore-imag-sign
@@ -1800,6 +1854,24 @@ sinh0023 sinh 0.043713693678420068 0.22512549887532657 -> 0.042624198673416713 0
 sinh0030 sinh 710.5 -2.3999999999999999 -> -1.3579970564885919e+308 -1.24394470907798e+308
 sinh0031 sinh -710.5 0.80000000000000004 -> -1.2830671601735164e+308 1.3210954193997678e+308
 
+-- Additional real values (mpmath)
+sinh0050 sinh 1e-100 0.0 -> 1.00000000000000002e-100 0.0
+sinh0051 sinh 5e-17 0.0 -> 4.9999999999999998955e-17 0.0
+sinh0052 sinh 1e-16 0.0 -> 9.999999999999999791e-17 0.0
+sinh0053 sinh 3.7e-08 0.0 -> 3.7000000000000008885e-8 0.0
+sinh0054 sinh 0.001 0.0 -> 0.0010000001666666750208 0.0
+sinh0055 sinh 0.2 0.0 -> 0.20133600254109399895 0.0
+sinh0056 sinh 1.0 0.0 -> 1.1752011936438014569 0.0
+sinh0057 sinh -3.7e-08 0.0 -> -3.7000000000000008885e-8 0.0
+sinh0058 sinh -0.001 0.0 -> -0.0010000001666666750208 0.0
+sinh0059 sinh -1.0 0.0 -> -1.1752011936438014569 0.0
+sinh0060 sinh 1.4436354751788103 0.0 -> 1.9999999999999999078 0.0
+sinh0061 sinh -1.4436354751788103 0.0 -> -1.9999999999999999078 0.0
+sinh0062 sinh 17.328679513998633 0.0 -> 16777215.999999992136 0.0
+sinh0063 sinh 18.714973875118524 0.0 -> 67108864.000000036211 0.0
+sinh0064 sinh 709.7827 0.0 -> 8.9883497833190073272e+307 0.0
+sinh0065 sinh -709.7827 0.0 -> -8.9883497833190073272e+307 0.0
+
 -- special values
 sinh1000 sinh 0.0 0.0 -> 0.0 0.0
 sinh1001 sinh 0.0 inf -> 0.0 nan        invalid ignore-real-sign
@@ -1897,6 +1969,24 @@ tanh0031 tanh -711 7.4000000000000004 -> -1.0 0.0
 tanh0032 tanh 1000 -2.3199999999999998 -> 1.0 0.0
 tanh0033 tanh -1.0000000000000001e+300 -9.6699999999999999 -> -1.0 -0.0
 
+-- Additional real values (mpmath)
+tanh0050 tanh 1e-100 0.0 -> 1.00000000000000002e-100 0.0
+tanh0051 tanh 5e-17 0.0 -> 4.9999999999999998955e-17 0.0
+tanh0052 tanh 1e-16 0.0 -> 9.999999999999999791e-17 0.0
+tanh0053 tanh 3.7e-08 0.0 -> 3.6999999999999983559e-8 0.0
+tanh0054 tanh 0.001 0.0 -> 0.00099999966666680002076 0.0
+tanh0055 tanh 0.2 0.0 -> 0.19737532022490401141 0.0
+tanh0056 tanh 1.0 0.0 -> 0.76159415595576488812 0.0
+tanh0057 tanh -3.7e-08 0.0 -> -3.6999999999999983559e-8 0.0
+tanh0058 tanh -0.001 0.0 -> -0.00099999966666680002076 0.0
+tanh0059 tanh -1.0 0.0 -> -0.76159415595576488812 0.0
+tanh0060 tanh 0.5493061443340549 0.0 -> 0.50000000000000003402 0.0
+tanh0061 tanh -0.5493061443340549 0.0 -> -0.50000000000000003402 0.0
+tanh0062 tanh 17.328679513998633 0.0 -> 0.99999999999999822364 0.0
+tanh0063 tanh 18.714973875118524 0.0 -> 0.99999999999999988898 0.0
+tanh0064 tanh 711 0.0 -> 1.0 0.0
+tanh0065 tanh 1.797e+308 0.0 -> 1.0 0.0
+
 --special values
 tanh1000 tanh 0.0 0.0 -> 0.0 0.0
 tanh1001 tanh 0.0 inf -> nan nan        invalid
@@ -1985,6 +2075,22 @@ cos0021 cos 4.8048375263775256 0.0062248852898515658 -> 0.092318702015846243 0.0
 cos0022 cos 7.9914515433858515 0.71659966615501436 -> -0.17375439906936566 -0.77217043527294582
 cos0023 cos 0.45124351152540226 1.6992693993812158 -> 2.543477948972237 -1.1528193694875477
 
+-- Additional real values (mpmath)
+cos0050 cos 1e-150 0.0 -> 1.0 -0.0
+cos0051 cos 1e-18 0.0 -> 1.0 -0.0
+cos0052 cos 1e-09 0.0 -> 0.9999999999999999995 -0.0
+cos0053 cos 0.0003 0.0 -> 0.9999999550000003375 -0.0
+cos0054 cos 0.2 0.0 -> 0.98006657784124162892 -0.0
+cos0055 cos 1.0 0.0 -> 0.5403023058681397174 -0.0
+cos0056 cos -1e-18 0.0 -> 1.0 0.0
+cos0057 cos -0.0003 0.0 -> 0.9999999550000003375 0.0
+cos0058 cos -1.0 0.0 -> 0.5403023058681397174 0.0
+cos0059 cos 1.0471975511965976 0.0 -> 0.50000000000000009945 -0.0
+cos0060 cos 2.5707963267948966 0.0 -> -0.84147098480789647357 -0.0
+cos0061 cos -2.5707963267948966 0.0 -> -0.84147098480789647357 0.0
+cos0062 cos 7.225663103256523 0.0 -> 0.58778525229247407559 -0.0
+cos0063 cos -8.79645943005142 0.0 -> -0.80901699437494722255 0.0
+
 -- special values
 cos1000 cos -0.0 0.0 -> 1.0 0.0
 cos1001 cos -inf 0.0 -> nan 0.0 invalid ignore-imag-sign
@@ -2073,6 +2179,22 @@ sin0021 sin 1.4342727387492671 0.81361889790284347 -> 1.3370960060947923 0.12336
 sin0022 sin 1.1518087354403725 4.8597235966150558 -> 58.919141989603041 26.237003403758852
 sin0023 sin 0.00087773078406649192 34.792379211312095 -> 565548145569.38245 644329685822700.62
 
+-- Additional real values (mpmath)
+sin0050 sin 1e-100 0.0 -> 1.00000000000000002e-100 0.0
+sin0051 sin 3.7e-08 0.0 -> 3.6999999999999992001e-8 0.0
+sin0052 sin 0.001 0.0 -> 0.00099999983333334168748 0.0
+sin0053 sin 0.2 0.0 -> 0.19866933079506122634 0.0
+sin0054 sin 1.0 0.0 -> 0.84147098480789650665 0.0
+sin0055 sin -3.7e-08 0.0 -> -3.6999999999999992001e-8 0.0
+sin0056 sin -0.001 0.0 -> -0.00099999983333334168748 0.0
+sin0057 sin -1.0 0.0 -> -0.84147098480789650665 0.0
+sin0058 sin 0.5235987755982989 0.0 -> 0.50000000000000004642 0.0
+sin0059 sin -0.5235987755982989 0.0 -> -0.50000000000000004642 0.0
+sin0060 sin 2.6179938779914944 0.0 -> 0.49999999999999996018 -0.0
+sin0061 sin -2.6179938779914944 0.0 -> -0.49999999999999996018 -0.0
+sin0062 sin 7.225663103256523 0.0 -> 0.80901699437494673648 0.0
+sin0063 sin -8.79645943005142 0.0 -> -0.58778525229247340658 -0.0
+
 -- special values
 sin1000 sin -0.0 0.0 -> -0.0 0.0
 sin1001 sin -inf 0.0 -> nan 0.0 invalid ignore-imag-sign
@@ -2161,6 +2283,25 @@ tan0021 tan 1.7809617968443272 1.5052381702853379 -> -0.044066222118946903 1.093
 tan0022 tan 1.1615313900880577 1.7956298728647107 -> 0.041793186826390362 1.0375339546034792
 tan0023 tan 0.067014779477908945 5.8517361577457097 -> 2.2088639754800034e-06 0.9999836182420061
 
+-- Additional real values (mpmath)
+tan0050 tan 1e-100 0.0 -> 1.00000000000000002e-100 0.0
+tan0051 tan 3.7e-08 0.0 -> 3.7000000000000017328e-8 0.0
+tan0052 tan 0.001 0.0 -> 0.0010000003333334666875 0.0
+tan0053 tan 0.2 0.0 -> 0.20271003550867249488 0.0
+tan0054 tan 1.0 0.0 -> 1.5574077246549022305 0.0
+tan0055 tan -3.7e-08 0.0 -> -3.7000000000000017328e-8 0.0
+tan0056 tan -0.001 0.0 -> -0.0010000003333334666875 0.0
+tan0057 tan -1.0 0.0 -> -1.5574077246549022305 0.0
+tan0058 tan 0.4636476090008061 0.0 -> 0.49999999999999997163 0.0
+tan0059 tan -0.4636476090008061 0.0 -> -0.49999999999999997163 0.0
+tan0060 tan 1.1071487177940904 0.0 -> 1.9999999999999995298 0.0
+tan0061 tan -1.1071487177940904 0.0 -> -1.9999999999999995298 0.0
+tan0062 tan 1.5 0.0 -> 14.101419947171719388 0.0
+tan0063 tan 1.57 0.0 -> 1255.7655915007896475 0.0
+tan0064 tan 1.5707963267948961 0.0 -> 1978937966095219.0538 0.0
+tan0065 tan 7.225663103256523 0.0 -> 1.3763819204711701522 0.0
+tan0066 tan -8.79645943005142 0.0 -> 0.7265425280053614098 0.0
+
 -- special values
 tan1000 tan -0.0 0.0 -> -0.0 0.0
 tan1001 tan -inf 0.0 -> nan nan invalid

Lib/test/test_math.py

@@ -29,6 +29,7 @@ test_dir = os.path.dirname(file) or os.curdir
 math_testcases = os.path.join(test_dir, 'math_testcases.txt')
 test_file = os.path.join(test_dir, 'cmath_testcases.txt')
 
+
 def to_ulps(x):
     """Convert a non-NaN float x to an integer, in such a way that
     adjacent floats are converted to adjacent integers.  Then
@@ -36,25 +37,39 @@ def to_ulps(x):
     floats.
 
     The results from this function will only make sense on platforms
-    where C doubles are represented in IEEE 754 binary64 format.
+    where native doubles are represented in IEEE 754 binary64 format.
 
+    Note: 0.0 and -0.0 are converted to 0 and -1, respectively.
     """
     n = struct.unpack('<q', struct.pack('<d', x))[0]
     if n < 0:
         n = ~(n+2**63)
     return n
 
-def ulps_check(expected, got, ulps=20):
-    """Given non-NaN floats `expected` and `got`,
-    check that they're equal to within the given number of ulps.
 
-    Returns None on success and an error message on failure."""
+def ulp(x):
+    """Return the value of the least significant bit of a
+    float x, such that the first float bigger than x is x+ulp(x).
+    Then, given an expected result x and a tolerance of n ulps,
+    the result y should be such that abs(y-x) <= n * ulp(x).
+    The results from this function will only make sense on platforms
+    where native doubles are represented in IEEE 754 binary64 format.
+    """
+    x = abs(float(x))
+    if math.isnan(x) or math.isinf(x):
+        return x
 
-    ulps_error = to_ulps(got) - to_ulps(expected)
-    if abs(ulps_error) <= ulps:
-        return None
-    return "error = {} ulps; permitted error = {} ulps".format(ulps_error,
-                                                               ulps)
+    # Find next float up from x.
+    n = struct.unpack('<q', struct.pack('<d', x))[0]
+    x_next = struct.unpack('<d', struct.pack('<q', n + 1))[0]
+    if math.isinf(x_next):
+        # Corner case: x was the largest finite float. Then it's
+        # not an exact power of two, so we can take the difference
+        # between x and the previous float.
+        x_prev = struct.unpack('<d', struct.pack('<q', n - 1))[0]
+        return x - x_prev
+    else:
+        return x_next - x
 
 # Here's a pure Python version of the math.factorial algorithm, for
 # documentation and comparison purposes.
@@ -106,24 +121,23 @@ def py_factorial(n):
         outer *= inner
     return outer << (n - count_set_bits(n))
 
-def acc_check(expected, got, rel_err=2e-15, abs_err = 5e-323):
-    """Determine whether non-NaN floats a and b are equal to within a
-    (small) rounding error.  The default values for rel_err and
-    abs_err are chosen to be suitable for platforms where a float is
-    represented by an IEEE 754 double.  They allow an error of between
-    9 and 19 ulps."""
+def ulp_abs_check(expected, got, ulp_tol, abs_tol):
+    """Given finite floats `expected` and `got`, check that they're
+    approximately equal to within the given number of ulps or the
+    given absolute tolerance, whichever is bigger.
 
-    # need to special case infinities, since inf - inf gives nan
-    if math.isinf(expected) and got == expected:
-        return None
-
-    error = got - expected
+    Returns None on success and an error message on failure.
+    """
+    ulp_error = abs(to_ulps(expected) - to_ulps(got))
+    abs_error = abs(expected - got)
 
-    permitted_error = max(abs_err, rel_err * abs(expected))
-    if abs(error) < permitted_error:
+    # Succeed if either abs_error <= abs_tol or ulp_error <= ulp_tol.
+    if abs_error <= abs_tol or ulp_error <= ulp_tol:
         return None
-    return "error = {}; permitted error = {}".format(error,
-                                                     permitted_error)
+    else:
+        fmt = ("error = {:.3g} ({:d} ulps); "
+               "permitted error = {:.3g} or {:d} ulps")
+        return fmt.format(abs_error, ulp_error, abs_tol, ulp_tol)
 
 def parse_mtestfile(fname):
     """Parse a file with test values
@@ -150,6 +164,7 @@ def parse_mtestfile(fname):
 
             yield (id, fn, float(arg), float(exp), flags)
 
+
 def parse_testfile(fname):
     """Parse a file with test values
 
@@ -171,8 +186,53 @@ def parse_testfile(fname):
             yield (id, fn,
                    float(arg_real), float(arg_imag),
                    float(exp_real), float(exp_imag),
-                   flags
-                  )
+                   flags)
+
+
+def result_check(expected, got, ulp_tol=5, abs_tol=0.0):
+    # Common logic of MathTests.(ftest, test_testcases, test_mtestcases)
+    """Compare arguments expected and got, as floats, if either
+    is a float, using a tolerance expressed in multiples of
+    ulp(expected) or absolutely (if given and greater).
+
+    As a convenience, when neither argument is a float, and for
+    non-finite floats, exact equality is demanded. Also, nan==nan
+    as far as this function is concerned.
+
+    Returns None on success and an error message on failure.
+    """
+
+    # Check exactly equal (applies also to strings representing exceptions)
+    if got == expected:
+        return None
+
+    failure = "not equal"
+
+    # Turn mixed float and int comparison (e.g. floor()) to all-float
+    if isinstance(expected, float) and isinstance(got, int):
+        got = float(got)
+    elif isinstance(got, float) and isinstance(expected, int):
+        expected = float(expected)
+
+    if isinstance(expected, float) and isinstance(got, float):
+        if math.isnan(expected) and math.isnan(got):
+            # Pass, since both nan
+            failure = None
+        elif math.isinf(expected) or math.isinf(got):
+            # We already know they're not equal, drop through to failure
+            pass
+        else:
+            # Both are finite floats (now). Are they close enough?
+            failure = ulp_abs_check(expected, got, ulp_tol, abs_tol)
+
+    # arguments are not equal, and if numeric, are too far apart
+    if failure is not None:
+        fail_fmt = "expected {!r}, got {!r}"
+        fail_msg = fail_fmt.format(expected, got)
+        fail_msg += ' ({})'.format(failure)
+        return fail_msg
+    else:
+        return None
 
 # Class providing an __index__ method.
 class MyIndexable(object):
@@ -184,18 +244,23 @@ class MyIndexable(object):
 
 class MathTests(unittest.TestCase):
 
-    def ftest(self, name, value, expected):
-        if abs(value-expected) > eps:
-            # Use %r instead of %f so the error message
-            # displays full precision. Otherwise discrepancies
-            # in the last few bits will lead to very confusing
-            # error messages
-            self.fail('%s returned %r, expected %r' %
-                      (name, value, expected))
+    def ftest(self, name, got, expected, ulp_tol=5, abs_tol=0.0):
+        """Compare arguments expected and got, as floats, if either
+        is a float, using a tolerance expressed in multiples of
+        ulp(expected) or absolutely, whichever is greater.
+
+        As a convenience, when neither argument is a float, and for
+        non-finite floats, exact equality is demanded. Also, nan==nan
+        in this function.
+        """
+        failure = result_check(expected, got, ulp_tol, abs_tol)
+        if failure is not None:
+            self.fail("{}: {}".format(name, failure))
 
     def testConstants(self):
-        self.ftest('pi', math.pi, 3.1415926)
-        self.ftest('e', math.e, 2.7182818)
+        # Ref: Abramowitz & Stegun (Dover, 1965)
+        self.ftest('pi', math.pi, 3.141592653589793238462643)
+        self.ftest('e', math.e, 2.718281828459045235360287)
         self.assertEqual(math.tau, 2*math.pi)
 
     def testAcos(self):
@@ -378,9 +443,9 @@ class MathTests(unittest.TestCase):
 
     def testCos(self):
         self.assertRaises(TypeError, math.cos)
-        self.ftest('cos(-pi/2)', math.cos(-math.pi/2), 0)
+        self.ftest('cos(-pi/2)', math.cos(-math.pi/2), 0, abs_tol=ulp(1))
         self.ftest('cos(0)', math.cos(0), 1)
-        self.ftest('cos(pi/2)', math.cos(math.pi/2), 0)
+        self.ftest('cos(pi/2)', math.cos(math.pi/2), 0, abs_tol=ulp(1))
         self.ftest('cos(pi)', math.cos(math.pi), -1)
         try:
             self.assertTrue(math.isnan(math.cos(INF)))
@@ -970,7 +1035,8 @@ class MathTests(unittest.TestCase):
     def testTanh(self):
         self.assertRaises(TypeError, math.tanh)
         self.ftest('tanh(0)', math.tanh(0), 0)
-        self.ftest('tanh(1)+tanh(-1)', math.tanh(1)+math.tanh(-1), 0)
+        self.ftest('tanh(1)+tanh(-1)', math.tanh(1)+math.tanh(-1), 0,
+                   abs_tol=ulp(1))
         self.ftest('tanh(inf)', math.tanh(INF), 1)
         self.ftest('tanh(-inf)', math.tanh(NINF), -1)
         self.assertTrue(math.isnan(math.tanh(NAN)))
@@ -1084,30 +1150,48 @@ class MathTests(unittest.TestCase):
 
     @requires_IEEE_754
     def test_testfile(self):
+        fail_fmt = "{}: {}({!r}): {}"
+
+        failures = []
         for id, fn, ar, ai, er, ei, flags in parse_testfile(test_file):
-            # Skip if either the input or result is complex, or if
-            # flags is nonempty
-            if ai != 0. or ei != 0. or flags:
+            # Skip if either the input or result is complex
+            if ai != 0.0 or ei != 0.0:
                 continue
             if fn in ['rect', 'polar']:
                 # no real versions of rect, polar
                 continue
+
             func = getattr(math, fn)
+
+            if 'invalid' in flags or 'divide-by-zero' in flags:
+                er = 'ValueError'
+            elif 'overflow' in flags:
+                er = 'OverflowError'
+
             try:
                 result = func(ar)
-            except ValueError as exc:
-                message = (("Unexpected ValueError: %s\n        " +
-                           "in test %s:%s(%r)\n") % (exc.args[0], id, fn, ar))
-                self.fail(message)
+            except ValueError:
+                result = 'ValueError'
             except OverflowError:
-                message = ("Unexpected OverflowError in " +
-                           "test %s:%s(%r)\n" % (id, fn, ar))
-                self.fail(message)
-            self.ftest("%s:%s(%r)" % (id, fn, ar), result, er)
+                result = 'OverflowError'
+
+            # Default tolerances
+            ulp_tol, abs_tol = 5, 0.0
+
+            failure = result_check(er, result, ulp_tol, abs_tol)
+            if failure is None:
+                continue
+
+            msg = fail_fmt.format(id, fn, ar, failure)
+            failures.append(msg)
+
+        if failures:
+            self.fail('Failures in test_testfile:\n  ' +
+                      '\n  '.join(failures))
 
     @requires_IEEE_754
     def test_mtestfile(self):
-        fail_fmt = "{}:{}({!r}): expected {!r}, got {!r}"
+        fail_fmt = "{}: {}({!r}): {}"
 
         failures = []
         for id, fn, arg, expected, flags in parse_mtestfile(math_testcases):
@@ -1125,41 +1209,48 @@ class MathTests(unittest.TestCase):
             except OverflowError:
                 got = 'OverflowError'
 
-            accuracy_failure = None
-            if isinstance(got, float) and isinstance(expected, float):
-                if math.isnan(expected) and math.isnan(got):
-                    continue
-                if not math.isnan(expected) and not math.isnan(got):
-                    if fn == 'lgamma':
-                        # we use a weaker accuracy test for lgamma;
-                        # lgamma only achieves an absolute error of
-                        # a few multiples of the machine accuracy, in
-                        # general.
-                        accuracy_failure = acc_check(expected, got,
-                                                  rel_err = 5e-15,
-                                                  abs_err = 5e-15)
-                    elif fn == 'erfc':
-                        # erfc has less-than-ideal accuracy for large
-                        # arguments (x ~ 25 or so), mainly due to the
-                        # error involved in computing exp(-x*x).
-                        #
-                        # XXX Would be better to weaken this test only
-                        # for large x, instead of for all x.
-                        accuracy_failure = ulps_check(expected, got, 2000)
-
-                    else:
-                        accuracy_failure = ulps_check(expected, got, 20)
-                    if accuracy_failure is None:
-                        continue
-
-            if isinstance(got, str) and isinstance(expected, str):
-                if got == expected:
-                    continue
-
-            fail_msg = fail_fmt.format(id, fn, arg, expected, got)
-            if accuracy_failure is not None:
-                fail_msg += ' ({})'.format(accuracy_failure)
-            failures.append(fail_msg)
+            # Default tolerances
+            ulp_tol, abs_tol = 5, 0.0
+
+            # Exceptions to the defaults
+            if fn == 'gamma':
+                # Experimental results on one platform gave
+                # an accuracy of <= 10 ulps across the entire float
+                # domain. We weaken that to require 20 ulp accuracy.
+                ulp_tol = 20
+
+            elif fn == 'lgamma':
+                # we use a weaker accuracy test for lgamma;
+                # lgamma only achieves an absolute error of
+                # a few multiples of the machine accuracy, in
+                # general.
+                abs_tol = 1e-15
+
+            elif fn == 'erfc' and arg >= 0.0:
+                # erfc has less-than-ideal accuracy for large
+                # arguments (x ~ 25 or so), mainly due to the
+                # error involved in computing exp(-x*x).
+                #
+                # Observed between CPython and mpmath at 25 dp:
+                #       x <  0 : err <= 2 ulp
+                #  0 <= x <  1 : err <= 10 ulp
+                #  1 <= x < 10 : err <= 100 ulp
+                # 10 <= x < 20 : err <= 300 ulp
+                # 20 <= x      : < 600 ulp
+                #
+                if arg < 1.0:
+                    ulp_tol = 10
+                elif arg < 10.0:
+                    ulp_tol = 100
+                else:
+                    ulp_tol = 1000
+
+            failure = result_check(expected, got, ulp_tol, abs_tol)
+            if failure is None:
+                continue
+
+            msg = fail_fmt.format(id, fn, arg, failure)
+            failures.append(msg)
 
         if failures:
             self.fail('Failures in test_mtestfile:\n  ' +

Misc/NEWS

@@ -135,6 +135,9 @@ Library
 Tests
 -----
 
+- Issue #26040: Improve test_math and test_cmath coverage and rigour. Patch by
+  Jeff Allen.
+
 - Issue #27787: Call gc.collect() before checking each test for "dangling
   threads", since the dangling threads are weak references.