proper overflow checks for mymemreplace (closes #24708)

Lib/test/test_strop.py

@@ -141,6 +141,11 @@ class StropFunctionTestCase(unittest.TestCase):
         else:
             self.assertEqual(len(r), len(a) * 3)
 
+    @unittest.skipUnless(sys.maxsize == 2147483647, "only for 32-bit")
+    def test_stropreplace_overflow(self):
+        a = "A" * 0x10000
+        self.assertRaises(MemoryError, strop.replace, a, "A", a)
+
 transtable = '\000\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037 !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`xyzdefghijklmnopqrstuvwxyz{|}~\177\200\201\202\203\204\205\206\207\210\211\212\213\214\215\216\217\220\221\222\223\224\225\226\227\230\231\232\233\234\235\236\237\240\241\242\243\244\245\246\247\250\251\252\253\254\255\256\257\260\261\262\263\264\265\266\267\270\271\272\273\274\275\276\277\300\301\302\303\304\305\306\307\310\311\312\313\314\315\316\317\320\321\322\323\324\325\326\327\330\331\332\333\334\335\336\337\340\341\342\343\344\345\346\347\350\351\352\353\354\355\356\357\360\361\362\363\364\365\366\367\370\371\372\373\374\375\376\377'
 
 

Misc/NEWS

@@ -34,6 +34,8 @@ Core and Builtins
 Library
 -------
 
+- Issue #24708: Fix possible integer overflow in strop.replace().
+
 - Issue #24620: Random.setstate() now validates the value of state last element.
 
 - Issue #13938: 2to3 converts StringTypes to a tuple. Patch from Mark Hammond.

Modules/stropmodule.c

@@ -1094,7 +1094,7 @@ mymemreplace(const char *str, Py_ssize_t len,           /* input string */
 {
     char *out_s;
     char *new_s;
-    Py_ssize_t nfound, offset, new_len;
+    Py_ssize_t nfound, offset, new_len, delta_len, abs_delta;
 
     if (len == 0 || pat_len > len)
         goto return_same;
@@ -1108,7 +1108,14 @@ mymemreplace(const char *str, Py_ssize_t len,           /* input string */
     if (nfound == 0)
         goto return_same;
 
-    new_len = len + nfound*(sub_len - pat_len);
+    delta_len = sub_len - pat_len;
+    abs_delta = (delta_len < 0) ? -delta_len : delta_len;
+    if (PY_SSIZE_T_MAX/nfound < abs_delta)
+        return NULL;
+    delta_len *= nfound;
+    if (PY_SSIZE_T_MAX - len < delta_len)
+        return NULL;
+    new_len = len + delta_len;
     if (new_len == 0) {
         /* Have to allocate something for the caller to free(). */
         out_s = (char *)PyMem_MALLOC(1);