Issue #25384: Fix binascii.rledecode_hqx()

Fix usage of _PyBytesWriter API. Use the new _PyBytesWriter_Resize() function
instead of _PyBytesWriter_Prepare().

Lib/test/test_binascii.py

@@ -159,11 +159,25 @@ class BinASCIITest(unittest.TestCase):
         # Then calculate the hexbin4 binary-to-ASCII translation
         rle = binascii.rlecode_hqx(self.data)
         a = binascii.b2a_hqx(self.type2test(rle))
+
         b, _ = binascii.a2b_hqx(self.type2test(a))
         res = binascii.rledecode_hqx(b)
-
         self.assertEqual(res, self.rawdata)
 
+    def test_rle(self):
+        # test repetition with a repetition longer than the limit of 255
+        data = (b'a' * 100 + b'b' + b'c' * 300)
+
+        encoded = binascii.rlecode_hqx(data)
+        self.assertEqual(encoded,
+                         (b'a\x90d'      # 'a' * 100
+                          b'b'           # 'b'
+                          b'c\x90\xff'   # 'c' * 255
+                          b'c\x90-'))    # 'c' * 45
+
+        decoded = binascii.rledecode_hqx(encoded)
+        self.assertEqual(decoded, data)
+
     def test_hex(self):
         # test hexlification
         s = b'{s\005\000\000\000worldi\002\000\000\000s\005\000\000\000helloi\001\000\000\0000'

Modules/binascii.c

@@ -800,14 +800,15 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data)
         return PyErr_NoMemory();
 
     /* Allocate a buffer of reasonable size. Resized when needed */
-    out_len = in_len * 2;
+    out_len = in_len;
     out_data = _PyBytesWriter_Alloc(&writer, out_len);
     if (out_data == NULL)
         return NULL;
 
     /* Use overallocation */
     writer.overallocate = 1;
-    out_len_left = writer.allocated;
+    out_len = writer.allocated;
+    out_len_left = out_len;
 
     /*
     ** We need two macros here to get/put bytes and handle
@@ -830,10 +831,12 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data)
                     overallocate the buffer anymore */                  \
                  writer.overallocate = 0;                               \
              }                                                          \
-             out_data = _PyBytesWriter_Prepare(&writer, out_data, 1);   \
+             out_data = _PyBytesWriter_Resize(&writer, out_data,        \
+                                              writer.allocated + 1);    \
              if (out_data == NULL)                                      \
                  goto error;                                            \
-             out_len_left = writer.allocated;                           \
+             out_len_left = writer.allocated - out_len - 1;             \
+             out_len = writer.allocated;                                \
          }                                                              \
          *out_data++ = b;                                               \
     } while(0)