Issue #4580: slicing of memoryviews when itemsize != 1 is wrong.

Also fix len() to return number of items rather than length in bytes.

I'm sorry it was not possible for me to work on this without reindenting
a bit some stuff around. The indentation in memoryobject.c is a mess,
I'll open a separate bug for it.

Include/object.h

@@ -153,6 +153,8 @@ typedef struct bufferinfo {
 	Py_ssize_t *shape;
 	Py_ssize_t *strides;
 	Py_ssize_t *suboffsets;
+	Py_ssize_t smalltable[2];  /* static store for shape and strides of
+				      mono-dimensional buffers. */
 	void *internal;
 } Py_buffer;
 

Lib/ctypes/test/test_pep3118.py

@@ -25,7 +25,10 @@ class Test(unittest.TestCase):
             v = memoryview(ob)
             try:
                 self.failUnlessEqual(normalize(v.format), normalize(fmt))
-                self.failUnlessEqual(len(v), sizeof(ob))
+                if shape is not None:
+                    self.failUnlessEqual(len(v), shape[0])
+                else:
+                    self.failUnlessEqual(len(v) * sizeof(itemtp), sizeof(ob))
                 self.failUnlessEqual(v.itemsize, sizeof(itemtp))
                 self.failUnlessEqual(v.shape, shape)
                 # ctypes object always have a non-strided memory block
@@ -37,7 +40,7 @@ class Test(unittest.TestCase):
                     n = 1
                     for dim in v.shape:
                         n = n * dim
-                    self.failUnlessEqual(v.itemsize * n, len(v))
+                    self.failUnlessEqual(n * v.itemsize, len(v.tobytes()))
             except:
                 # so that we can see the failing type
                 print(tp)
@@ -49,7 +52,10 @@ class Test(unittest.TestCase):
             v = memoryview(ob)
             try:
                 self.failUnlessEqual(v.format, fmt)
-                self.failUnlessEqual(len(v), sizeof(ob))
+                if shape is not None:
+                    self.failUnlessEqual(len(v), shape[0])
+                else:
+                    self.failUnlessEqual(len(v) * sizeof(itemtp), sizeof(ob))
                 self.failUnlessEqual(v.itemsize, sizeof(itemtp))
                 self.failUnlessEqual(v.shape, shape)
                 # ctypes object always have a non-strided memory block
@@ -61,7 +67,7 @@ class Test(unittest.TestCase):
                     n = 1
                     for dim in v.shape:
                         n = n * dim
-                    self.failUnlessEqual(v.itemsize * n, len(v))
+                    self.failUnlessEqual(n, len(v))
             except:
                 # so that we can see the failing type
                 print(tp)

Lib/test/test_io.py

@@ -260,7 +260,7 @@ class IOTest(unittest.TestCase):
 
     def test_array_writes(self):
         a = array.array('i', range(10))
-        n = len(memoryview(a))
+        n = len(a.tostring())
         f = io.open(support.TESTFN, "wb", 0)
         self.assertEqual(f.write(a), n)
         f.close()

Lib/test/test_sys.py

@@ -559,7 +559,7 @@ class SizeofTest(unittest.TestCase):
         check(32768*32768-1, size(vh) + 2*self.H)
         check(32768*32768, size(vh) + 3*self.H)
         # memory
-        check(memoryview(b''), size(h + 'P PP2P2i5P'))
+        check(memoryview(b''), size(h + 'P PP2P2i7P'))
         # module
         check(unittest, size(h + '3P'))
         # None

Misc/NEWS

@@ -12,6 +12,10 @@ What's New in Python 3.1 alpha 0
 Core and Builtins
 -----------------
 
+- Issue #4580: Fix slicing of memoryviews when the item size is greater than
+  one byte. Also fixes the meaning of len() so that it returns the number of
+  items, rather than the size in bytes.
+
 - Issue #4075: Use OutputDebugStringW in Py_FatalError.
 
 - Issue #4747: When the terminal does not use utf-8, executing a script with

Objects/memoryobject.c

@@ -3,26 +3,31 @@
 
 #include "Python.h"
 
-static void
-dup_buffer(Py_buffer *dest, Py_buffer *src)
-{
-	*dest = *src;
-        if (src->shape == &(src->len))
-            dest->shape = &(dest->len);
-        if (src->strides == &(src->itemsize))
-            dest->strides = &(dest->itemsize);
-}
-
-/* XXX The buffer API should mandate that the shape array be non-NULL, but
-   it would complicate some code since the (de)allocation semantics of shape
-   are not specified. */
 static Py_ssize_t
 get_shape0(Py_buffer *buf)
 {
     if (buf->shape != NULL)
         return buf->shape[0];
-    assert(buf->ndim == 1 && buf->itemsize > 0);
-    return buf->len / buf->itemsize;
+    if (buf->ndim == 0)
+        return 1;
+    PyErr_SetString(PyExc_TypeError,
+        "exported buffer does not have any shape information associated "
+        "to it");
+    return -1;
+}
+
+static void
+dup_buffer(Py_buffer *dest, Py_buffer *src)
+{
+    *dest = *src;
+    if (src->ndim == 1 && src->shape != NULL) {
+        dest->shape = &(dest->smalltable[0]);
+        dest->shape[0] = get_shape0(src);
+    }
+    if (src->ndim == 1 && src->strides != NULL) {
+        dest->strides = &(dest->smalltable[1]);
+        dest->strides[0] = src->strides[0];
+    }
 }
 
 static int
@@ -449,8 +454,6 @@ memory_tolist(PyMemoryViewObject *mem, PyObject *noargs)
 	return res;
 }
 
-
-
 static PyMethodDef memory_methods[] = {
         {"tobytes", (PyCFunction)memory_tobytes, METH_NOARGS, NULL},
         {"tolist", (PyCFunction)memory_tolist, METH_NOARGS, NULL},
@@ -512,16 +515,10 @@ memory_str(PyMemoryViewObject *self)
 }
 
 /* Sequence methods */
-
 static Py_ssize_t
 memory_length(PyMemoryViewObject *self)
 {
-        Py_buffer view;
-
-        if (PyObject_GetBuffer((PyObject *)self, &view, PyBUF_FULL) < 0)
-                return -1;
-        PyBuffer_Release(&view);
-	return view.len;
+    return get_shape0(&self->view);
 }
 
 /*
@@ -605,19 +602,17 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key)
     
             /* XXX There should be an API to create a subbuffer */
             if (view->obj != NULL) {
-			    if (PyObject_GetBuffer(view->obj,
-					    &newview, newflags) == -1)
+                if (PyObject_GetBuffer(view->obj, &newview, newflags) == -1)
                     return NULL;
             }
             else {
                 newview = *view;
             }
             newview.buf = newbuf;
-		    newview.len = slicelength;
+            newview.len = slicelength * newview.itemsize;
             newview.format = view->format;
-		    if (view->shape == &(view->len))
-			    newview.shape = &(newview.len);
-		    if (view->strides == &(view->itemsize))
+            newview.shape = &(newview.smalltable[0]);
+            newview.shape[0] = slicelength;
             newview.strides = &(newview.itemsize);
             return PyMemoryView_FromBuffer(&newview);
         }
@@ -747,7 +742,7 @@ memory_richcompare(PyObject *v, PyObject *w, int op)
 	if (vv.itemsize != ww.itemsize || vv.len != ww.len)
 		goto _end;
 
-	equal = !memcmp(vv.buf, ww.buf, vv.len * vv.itemsize);
+	equal = !memcmp(vv.buf, ww.buf, vv.len);
 
 _end:
 	PyBuffer_Release(&vv);

Objects/unicodeobject.c

@@ -1205,7 +1205,7 @@ PyObject *PyUnicode_Decode(const char *s,
 
     /* Decode via the codec registry */
     buffer = NULL;
-    if (PyBuffer_FillInfo(&info, NULL, (void *)s, size, 1, PyBUF_SIMPLE) < 0)
+    if (PyBuffer_FillInfo(&info, NULL, (void *)s, size, 1, PyBUF_FULL_RO) < 0)
         goto onError;
     buffer = PyMemoryView_FromBuffer(&info);
     if (buffer == NULL)