Allow long objects as a position value of error callbacks returned.

Lib/test/test_multibytecodec.py

@@ -7,7 +7,7 @@
 
 from test import test_support
 from test import test_multibytecodec_support
-import unittest, StringIO, codecs
+import unittest, StringIO, codecs, sys
 
 class Test_MultibyteCodec(unittest.TestCase):
 
@@ -19,6 +19,12 @@ class Test_MultibyteCodec(unittest.TestCase):
     def test_str_decode(self):
         self.assertEqual('abcd'.encode('gb18030'), 'abcd')
 
+    def test_errorcallback_longindex(self):
+        dec = codecs.getdecoder('euc-kr')
+        myreplace  = lambda exc: (u'', sys.maxint+1)
+        codecs.register_error('test.cjktest', myreplace)
+        self.assertRaises(IndexError, dec,
+                          'apple\x92ham\x93spam', 'test.cjktest')
 
 class Test_IncrementalEncoder(unittest.TestCase):
 

Lib/test/test_multibytecodec_support.py

@@ -60,7 +60,7 @@ class TestBase:
             "ଓଣୠ nd eggs"
         )
 
-    def test_customreplace(self):
+    def test_customreplace_encode(self):
         if self.has_iso10646:
             return
 
@@ -96,6 +96,19 @@ class TestBase:
             self.assertRaises(TypeError, self.encode, self.unmappedunicode,
                               'test.cjktest')
 
+    def test_callback_long_index(self):
+        def myreplace(exc):
+            return (u'x', long(exc.end))
+        codecs.register_error("test.cjktest", myreplace)
+        self.assertEqual(self.encode(u'abcd' + self.unmappedunicode + u'efgh',
+                                     'test.cjktest'), ('abcdxefgh', 9))
+
+        def myreplace(exc):
+            return (u'x', sys.maxint + 1)
+        codecs.register_error("test.cjktest", myreplace)
+        self.assertRaises(IndexError, self.encode, self.unmappedunicode,
+                          'test.cjktest')
+
     def test_callback_None_index(self):
         def myreplace(exc):
             return (u'x', None)

Modules/cjkcodecs/multibytecodec.c

@@ -304,7 +304,8 @@ multibytecodec_encerror(MultibyteCodec *codec,
 
 	if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||
 	    !PyUnicode_Check((tobj = PyTuple_GET_ITEM(retobj, 0))) ||
-	    !PyInt_Check(PyTuple_GET_ITEM(retobj, 1))) {
+	    !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) ||
+	      PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) {
 		PyErr_SetString(PyExc_TypeError,
 				"encoding error handler must return "
 				"(unicode, int) tuple");
@@ -328,12 +329,13 @@ multibytecodec_encerror(MultibyteCodec *codec,
 	buf->outbuf += retstrsize;
 
 	newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
-	if (newpos < 0)
+	if (newpos < 0 && !PyErr_Occurred())
 		newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);
 	if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {
+		PyErr_Clear();
 		PyErr_Format(PyExc_IndexError,
-			     "position %d from error handler out of bounds",
-			     (int)newpos);
+			     "position %ld from error handler out of bounds",
+			     (long)newpos);
 		goto errorexit;
 	}
 	buf->inbuf = buf->inbuf_top + newpos;
@@ -421,7 +423,8 @@ multibytecodec_decerror(MultibyteCodec *codec,
 
 	if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||
 	    !PyUnicode_Check((retuni = PyTuple_GET_ITEM(retobj, 0))) ||
-	    !PyInt_Check(PyTuple_GET_ITEM(retobj, 1))) {
+	    !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) ||
+	      PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) {
 		PyErr_SetString(PyExc_TypeError,
 				"decoding error handler must return "
 				"(unicode, int) tuple");
@@ -437,12 +440,13 @@ multibytecodec_decerror(MultibyteCodec *codec,
 	}
 
 	newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
-	if (newpos < 0)
+	if (newpos < 0 && !PyErr_Occurred())
 		newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);
 	if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {
+		PyErr_Clear();
 		PyErr_Format(PyExc_IndexError,
-				"position %d from error handler out of bounds",
-				(int)newpos);
+				"position %ld from error handler out of bounds",
+				(long)newpos);
 		goto errorexit;
 	}
 	buf->inbuf = buf->inbuf_top + newpos;