bpo-36785: PEP 574 implementation (GH-7076)

Include/Python.h

@@ -124,6 +124,7 @@
 #include "weakrefobject.h"
 #include "structseq.h"
 #include "namespaceobject.h"
+#include "picklebufobject.h"
 
 #include "codecs.h"
 #include "pyerrors.h"

Include/picklebufobject.h

+/* PickleBuffer object. This is built-in for ease of use from third-party
+ * C extensions.
+ */
+
+#ifndef Py_PICKLEBUFOBJECT_H
+#define Py_PICKLEBUFOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_LIMITED_API
+
+PyAPI_DATA(PyTypeObject) PyPickleBuffer_Type;
+
+#define PyPickleBuffer_Check(op) (Py_TYPE(op) == &PyPickleBuffer_Type)
+
+/* Create a PickleBuffer redirecting to the given buffer-enabled object */
+PyAPI_FUNC(PyObject *) PyPickleBuffer_FromObject(PyObject *);
+/* Get the PickleBuffer's underlying view to the original object
+ * (NULL if released)
+ */
+PyAPI_FUNC(const Py_buffer *) PyPickleBuffer_GetBuffer(PyObject *);
+/* Release the PickleBuffer.  Returns 0 on success, -1 on error. */
+PyAPI_FUNC(int) PyPickleBuffer_Release(PyObject *);
+
+#endif /* !Py_LIMITED_API */
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_PICKLEBUFOBJECT_H */

Lib/pickletools.py

@@ -565,6 +565,41 @@ bytes8 = ArgumentDescriptor(
               the number of bytes, and the second argument is that many bytes.
               """)
 
+
+def read_bytearray8(f):
+    r"""
+    >>> import io, struct, sys
+    >>> read_bytearray8(io.BytesIO(b"\x00\x00\x00\x00\x00\x00\x00\x00abc"))
+    bytearray(b'')
+    >>> read_bytearray8(io.BytesIO(b"\x03\x00\x00\x00\x00\x00\x00\x00abcdef"))
+    bytearray(b'abc')
+    >>> bigsize8 = struct.pack("<Q", sys.maxsize//3)
+    >>> read_bytearray8(io.BytesIO(bigsize8 + b"abcdef"))  #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ...
+    ValueError: expected ... bytes in a bytearray8, but only 6 remain
+    """
+
+    n = read_uint8(f)
+    assert n >= 0
+    if n > sys.maxsize:
+        raise ValueError("bytearray8 byte count > sys.maxsize: %d" % n)
+    data = f.read(n)
+    if len(data) == n:
+        return bytearray(data)
+    raise ValueError("expected %d bytes in a bytearray8, but only %d remain" %
+                     (n, len(data)))
+
+bytearray8 = ArgumentDescriptor(
+              name="bytearray8",
+              n=TAKEN_FROM_ARGUMENT8U,
+              reader=read_bytearray8,
+              doc="""A counted bytearray.
+
+              The first argument is an 8-byte little-endian unsigned int giving
+              the number of bytes, and the second argument is that many bytes.
+              """)
+
 def read_unicodestringnl(f):
     r"""
     >>> import io
@@ -970,6 +1005,11 @@ pybytes = StackObject(
     obtype=bytes,
     doc="A Python bytes object.")
 
+pybytearray = StackObject(
+    name='bytearray',
+    obtype=bytearray,
+    doc="A Python bytearray object.")
+
 pyunicode = StackObject(
     name='str',
     obtype=str,
@@ -1005,6 +1045,11 @@ pyfrozenset = StackObject(
     obtype=set,
     doc="A Python frozenset object.")
 
+pybuffer = StackObject(
+    name='buffer',
+    obtype=object,
+    doc="A Python buffer-like object.")
+
 anyobject = StackObject(
     name='any',
     obtype=object,
@@ -1265,7 +1310,7 @@ opcodes = [
       object instead.
       """),
 
-    # Bytes (protocol 3 only; older protocols don't support bytes at all)
+    # Bytes (protocol 3 and higher)
 
     I(name='BINBYTES',
       code='B',
@@ -1306,6 +1351,39 @@ opcodes = [
       which are taken literally as the string content.
       """),
 
+    # Bytearray (protocol 5 and higher)
+
+    I(name='BYTEARRAY8',
+      code='\x96',
+      arg=bytearray8,
+      stack_before=[],
+      stack_after=[pybytearray],
+      proto=5,
+      doc="""Push a Python bytearray object.
+
+      There are two arguments:  the first is an 8-byte unsigned int giving
+      the number of bytes in the bytearray, and the second is that many bytes,
+      which are taken literally as the bytearray content.
+      """),
+
+    # Out-of-band buffer (protocol 5 and higher)
+
+    I(name='NEXT_BUFFER',
+      code='\x97',
+      arg=None,
+      stack_before=[],
+      stack_after=[pybuffer],
+      proto=5,
+      doc="Push an out-of-band buffer object."),
+
+    I(name='READONLY_BUFFER',
+      code='\x98',
+      arg=None,
+      stack_before=[pybuffer],
+      stack_after=[pybuffer],
+      proto=5,
+      doc="Make an out-of-band buffer object read-only."),
+
     # Ways to spell None.
 
     I(name='NONE',

Lib/test/test_inspect.py

@@ -2894,16 +2894,15 @@ class TestSignatureObject(unittest.TestCase):
     @unittest.skipIf(MISSING_C_DOCSTRINGS,
                      "Signature information for builtins requires docstrings")
     def test_signature_on_builtin_class(self):
-        self.assertEqual(str(inspect.signature(_pickle.Pickler)),
-                         '(file, protocol=None, fix_imports=True)')
+        expected = ('(file, protocol=None, fix_imports=True, '
+                    'buffer_callback=None)')
+        self.assertEqual(str(inspect.signature(_pickle.Pickler)), expected)
 
         class P(_pickle.Pickler): pass
         class EmptyTrait: pass
         class P2(EmptyTrait, P): pass
-        self.assertEqual(str(inspect.signature(P)),
-                         '(file, protocol=None, fix_imports=True)')
-        self.assertEqual(str(inspect.signature(P2)),
-                         '(file, protocol=None, fix_imports=True)')
+        self.assertEqual(str(inspect.signature(P)), expected)
+        self.assertEqual(str(inspect.signature(P2)), expected)
 
         class P3(P2):
             def __init__(self, spam):

Lib/test/test_pickle.py

@@ -57,9 +57,9 @@ class PyPicklerTests(AbstractPickleTests):
     pickler = pickle._Pickler
     unpickler = pickle._Unpickler
 
-    def dumps(self, arg, proto=None):
+    def dumps(self, arg, proto=None, **kwargs):
         f = io.BytesIO()
-        p = self.pickler(f, proto)
+        p = self.pickler(f, proto, **kwargs)
         p.dump(arg)
         f.seek(0)
         return bytes(f.read())
@@ -78,8 +78,8 @@ class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests,
                         AttributeError, ValueError,
                         struct.error, IndexError, ImportError)
 
-    def dumps(self, arg, protocol=None):
-        return pickle.dumps(arg, protocol)
+    def dumps(self, arg, protocol=None, **kwargs):
+        return pickle.dumps(arg, protocol, **kwargs)
 
     def loads(self, buf, **kwds):
         return pickle.loads(buf, **kwds)
@@ -271,7 +271,7 @@ if has_c_implementation:
         check_sizeof = support.check_sizeof
 
         def test_pickler(self):
-            basesize = support.calcobjsize('6P2n3i2n3i2P')
+            basesize = support.calcobjsize('7P2n3i2n3i2P')
             p = _pickle.Pickler(io.BytesIO())
             self.assertEqual(object.__sizeof__(p), basesize)
             MT_size = struct.calcsize('3nP0n')
@@ -288,7 +288,7 @@ if has_c_implementation:
                 0)  # Write buffer is cleared after every dump().
 
         def test_unpickler(self):
-            basesize = support.calcobjsize('2P2n2P 2P2n2i5P 2P3n6P2n2i')
+            basesize = support.calcobjsize('2P2n2P 2P2n2i5P 2P3n8P2n2i')
             unpickler = _pickle.Unpickler
             P = struct.calcsize('P')  # Size of memo table entry.
             n = struct.calcsize('n')  # Size of mark table entry.

Lib/test/test_picklebuffer.py

+"""Unit tests for the PickleBuffer object.
+
+Pickling tests themselves are in pickletester.py.
+"""
+
+import gc
+from pickle import PickleBuffer
+import sys
+import weakref
+import unittest
+
+from test import support
+
+
+class B(bytes):
+    pass
+
+
+class PickleBufferTest(unittest.TestCase):
+
+    def check_memoryview(self, pb, equiv):
+        with memoryview(pb) as m:
+            with memoryview(equiv) as expected:
+                self.assertEqual(m.nbytes, expected.nbytes)
+                self.assertEqual(m.readonly, expected.readonly)
+                self.assertEqual(m.itemsize, expected.itemsize)
+                self.assertEqual(m.shape, expected.shape)
+                self.assertEqual(m.strides, expected.strides)
+                self.assertEqual(m.c_contiguous, expected.c_contiguous)
+                self.assertEqual(m.f_contiguous, expected.f_contiguous)
+                self.assertEqual(m.format, expected.format)
+                self.assertEqual(m.tobytes(), expected.tobytes())
+
+    def test_constructor_failure(self):
+        with self.assertRaises(TypeError):
+            PickleBuffer()
+        with self.assertRaises(TypeError):
+            PickleBuffer("foo")
+        # Released memoryview fails taking a buffer
+        m = memoryview(b"foo")
+        m.release()
+        with self.assertRaises(ValueError):
+            PickleBuffer(m)
+
+    def test_basics(self):
+        pb = PickleBuffer(b"foo")
+        self.assertEqual(b"foo", bytes(pb))
+        with memoryview(pb) as m:
+            self.assertTrue(m.readonly)
+
+        pb = PickleBuffer(bytearray(b"foo"))
+        self.assertEqual(b"foo", bytes(pb))
+        with memoryview(pb) as m:
+            self.assertFalse(m.readonly)
+            m[0] = 48
+        self.assertEqual(b"0oo", bytes(pb))
+
+    def test_release(self):
+        pb = PickleBuffer(b"foo")
+        pb.release()
+        with self.assertRaises(ValueError) as raises:
+            memoryview(pb)
+        self.assertIn("operation forbidden on released PickleBuffer object",
+                      str(raises.exception))
+        # Idempotency
+        pb.release()
+
+    def test_cycle(self):
+        b = B(b"foo")
+        pb = PickleBuffer(b)
+        b.cycle = pb
+        wpb = weakref.ref(pb)
+        del b, pb
+        gc.collect()
+        self.assertIsNone(wpb())
+
+    def test_ndarray_2d(self):
+        # C-contiguous
+        ndarray = support.import_module("_testbuffer").ndarray
+        arr = ndarray(list(range(12)), shape=(4, 3), format='<i')
+        self.assertTrue(arr.c_contiguous)
+        self.assertFalse(arr.f_contiguous)
+        pb = PickleBuffer(arr)
+        self.check_memoryview(pb, arr)
+        # Non-contiguous
+        arr = arr[::2]
+        self.assertFalse(arr.c_contiguous)
+        self.assertFalse(arr.f_contiguous)
+        pb = PickleBuffer(arr)
+        self.check_memoryview(pb, arr)
+        # F-contiguous
+        arr = ndarray(list(range(12)), shape=(3, 4), strides=(4, 12), format='<i')
+        self.assertTrue(arr.f_contiguous)
+        self.assertFalse(arr.c_contiguous)
+        pb = PickleBuffer(arr)
+        self.check_memoryview(pb, arr)
+
+    # Tests for PickleBuffer.raw()
+
+    def check_raw(self, obj, equiv):
+        pb = PickleBuffer(obj)
+        with pb.raw() as m:
+            self.assertIsInstance(m, memoryview)
+            self.check_memoryview(m, equiv)
+
+    def test_raw(self):
+        for obj in (b"foo", bytearray(b"foo")):
+            with self.subTest(obj=obj):
+                self.check_raw(obj, obj)
+
+    def test_raw_ndarray(self):
+        # 1-D, contiguous
+        ndarray = support.import_module("_testbuffer").ndarray
+        arr = ndarray(list(range(3)), shape=(3,), format='<h')
+        equiv = b"\x00\x00\x01\x00\x02\x00"
+        self.check_raw(arr, equiv)
+        # 2-D, C-contiguous
+        arr = ndarray(list(range(6)), shape=(2, 3), format='<h')
+        equiv = b"\x00\x00\x01\x00\x02\x00\x03\x00\x04\x00\x05\x00"
+        self.check_raw(arr, equiv)
+        # 2-D, F-contiguous
+        arr = ndarray(list(range(6)), shape=(2, 3), strides=(2, 4),
+                      format='<h')
+        # Note this is different from arr.tobytes()
+        equiv = b"\x00\x00\x01\x00\x02\x00\x03\x00\x04\x00\x05\x00"
+        self.check_raw(arr, equiv)
+        # 0-D
+        arr = ndarray(456, shape=(), format='<i')
+        equiv = b'\xc8\x01\x00\x00'
+        self.check_raw(arr, equiv)
+
+    def check_raw_non_contiguous(self, obj):
+        pb = PickleBuffer(obj)
+        with self.assertRaisesRegex(BufferError, "non-contiguous"):
+            pb.raw()
+
+    def test_raw_non_contiguous(self):
+        # 1-D
+        ndarray = support.import_module("_testbuffer").ndarray
+        arr = ndarray(list(range(6)), shape=(6,), format='<i')[::2]
+        self.check_raw_non_contiguous(arr)
+        # 2-D
+        arr = ndarray(list(range(12)), shape=(4, 3), format='<i')[::2]
+        self.check_raw_non_contiguous(arr)
+
+    def test_raw_released(self):
+        pb = PickleBuffer(b"foo")
+        pb.release()
+        with self.assertRaises(ValueError) as raises:
+            pb.raw()
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_pickletools.py

@@ -6,8 +6,8 @@ import unittest
 
 class OptimizedPickleTests(AbstractPickleTests):
 
-    def dumps(self, arg, proto=None):
-        return pickletools.optimize(pickle.dumps(arg, proto))
+    def dumps(self, arg, proto=None, **kwargs):
+        return pickletools.optimize(pickle.dumps(arg, proto, **kwargs))
 
     def loads(self, buf, **kwds):
         return pickle.loads(buf, **kwds)
@@ -71,23 +71,24 @@ class MiscTestCase(unittest.TestCase):
                      'read_uint8', 'read_stringnl', 'read_stringnl_noescape',
                      'read_stringnl_noescape_pair', 'read_string1',
                      'read_string4', 'read_bytes1', 'read_bytes4',
-                     'read_bytes8', 'read_unicodestringnl',
+                     'read_bytes8', 'read_bytearray8', 'read_unicodestringnl',
                      'read_unicodestring1', 'read_unicodestring4',
                      'read_unicodestring8', 'read_decimalnl_short',
                      'read_decimalnl_long', 'read_floatnl', 'read_float8',
                      'read_long1', 'read_long4',
                      'uint1', 'uint2', 'int4', 'uint4', 'uint8', 'stringnl',
                      'stringnl_noescape', 'stringnl_noescape_pair', 'string1',
-                     'string4', 'bytes1', 'bytes4', 'bytes8',
+                     'string4', 'bytes1', 'bytes4', 'bytes8', 'bytearray8',
                      'unicodestringnl', 'unicodestring1', 'unicodestring4',
                      'unicodestring8', 'decimalnl_short', 'decimalnl_long',
                      'floatnl', 'float8', 'long1', 'long4',
                      'StackObject',
                      'pyint', 'pylong', 'pyinteger_or_bool', 'pybool', 'pyfloat',
-                     'pybytes_or_str', 'pystring', 'pybytes', 'pyunicode',
-                     'pynone', 'pytuple', 'pylist', 'pydict', 'pyset',
-                     'pyfrozenset', 'anyobject', 'markobject', 'stackslice',
-                     'OpcodeInfo', 'opcodes', 'code2op',
+                     'pybytes_or_str', 'pystring', 'pybytes', 'pybytearray',
+                     'pyunicode', 'pynone', 'pytuple', 'pylist', 'pydict',
+                     'pyset', 'pyfrozenset', 'pybuffer', 'anyobject',
+                     'markobject', 'stackslice', 'OpcodeInfo', 'opcodes',
+                     'code2op',
                      }
         support.check__all__(self, pickletools, blacklist=blacklist)
 

Lib/test/test_pyclbr.py

@@ -224,7 +224,7 @@ class PyclbrTest(TestCase):
         # These were once about the 10 longest modules
         cm('random', ignore=('Random',))  # from _random import Random as CoreGenerator
         cm('cgi', ignore=('log',))      # set with = in module
-        cm('pickle', ignore=('partial',))
+        cm('pickle', ignore=('partial', 'PickleBuffer'))
         # TODO(briancurtin): openfp is deprecated as of 3.7.
         # Update this once it has been removed.
         cm('aifc', ignore=('openfp', '_aifc_params'))  # set with = in module

Makefile.pre.in

@@ -382,6 +382,7 @@ OBJECT_OBJS=	\
 		Objects/bytearrayobject.o \
 		Objects/bytesobject.o \
 		Objects/call.o \
+		Objects/capsule.o \
 		Objects/cellobject.o \
 		Objects/classobject.o \
 		Objects/codeobject.o \
@@ -406,7 +407,7 @@ OBJECT_OBJS=	\
 		Objects/namespaceobject.o \
 		Objects/object.o \
 		Objects/obmalloc.o \
-		Objects/capsule.o \
+		Objects/picklebufobject.o \
 		Objects/rangeobject.o \
 		Objects/setobject.o \
 		Objects/sliceobject.o \
@@ -1009,6 +1010,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/osdefs.h \
 		$(srcdir)/Include/osmodule.h \
 		$(srcdir)/Include/patchlevel.h \
+		$(srcdir)/Include/picklebufobject.h \
 		$(srcdir)/Include/pyarena.h \
 		$(srcdir)/Include/pycapsule.h \
 		$(srcdir)/Include/pyctype.h \

Misc/NEWS.d/next/Library/2019-05-03-20-47-55.bpo-36785.PQLnPq.rst

+Implement PEP 574 (pickle protocol 5 with out-of-band buffers).

Objects/object.c

@@ -1839,6 +1839,7 @@ _PyTypes_Init(void)
     INIT_TYPE(&PyMethodDescr_Type, "method descr");
     INIT_TYPE(&PyCallIter_Type, "call iter");
     INIT_TYPE(&PySeqIter_Type, "sequence iterator");
+    INIT_TYPE(&PyPickleBuffer_Type, "pickle.PickleBuffer");
     INIT_TYPE(&PyCoro_Type, "coroutine");
     INIT_TYPE(&_PyCoroWrapper_Type, "coroutine wrapper");
     INIT_TYPE(&_PyInterpreterID_Type, "interpreter ID");

Objects/picklebufobject.c

+/* PickleBuffer object implementation */
+
+#define PY_SSIZE_T_CLEAN
+#include "Python.h"
+#include <stddef.h>
+
+typedef struct {
+    PyObject_HEAD
+    /* The view exported by the original object */
+    Py_buffer view;
+    PyObject *weakreflist;
+} PyPickleBufferObject;
+
+/* C API */
+
+PyObject *
+PyPickleBuffer_FromObject(PyObject *base)
+{
+    PyTypeObject *type = &PyPickleBuffer_Type;
+    PyPickleBufferObject *self;
+
+    self = (PyPickleBufferObject *) type->tp_alloc(type, 0);
+    if (self == NULL) {
+        return NULL;
+    }
+    self->view.obj = NULL;
+    self->weakreflist = NULL;
+    if (PyObject_GetBuffer(base, &self->view, PyBUF_FULL_RO) < 0) {
+        Py_DECREF(self);
+        return NULL;
+    }
+    return (PyObject *) self;
+}
+
+const Py_buffer *
+PyPickleBuffer_GetBuffer(PyObject *obj)
+{
+    PyPickleBufferObject *self = (PyPickleBufferObject *) obj;
+
+    if (!PyPickleBuffer_Check(obj)) {
+        PyErr_Format(PyExc_TypeError,
+                     "expected PickleBuffer, %.200s found",
+                     Py_TYPE(obj)->tp_name);
+        return NULL;
+    }
+    if (self->view.obj == NULL) {
+        PyErr_SetString(PyExc_ValueError,
+                        "operation forbidden on released PickleBuffer object");
+        return NULL;
+    }
+    return &self->view;
+}
+
+int
+PyPickleBuffer_Release(PyObject *obj)
+{
+    PyPickleBufferObject *self = (PyPickleBufferObject *) obj;
+
+    if (!PyPickleBuffer_Check(obj)) {
+        PyErr_Format(PyExc_TypeError,
+                     "expected PickleBuffer, %.200s found",
+                     Py_TYPE(obj)->tp_name);
+        return -1;
+    }
+    PyBuffer_Release(&self->view);
+    return 0;
+}
+
+static PyObject *
+picklebuf_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+    PyPickleBufferObject *self;
+    PyObject *base;
+    char *keywords[] = {"", NULL};
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:PickleBuffer",
+                                     keywords, &base)) {
+        return NULL;
+    }
+
+    self = (PyPickleBufferObject *) type->tp_alloc(type, 0);
+    if (self == NULL) {
+        return NULL;
+    }
+    self->view.obj = NULL;
+    self->weakreflist = NULL;
+    if (PyObject_GetBuffer(base, &self->view, PyBUF_FULL_RO) < 0) {
+        Py_DECREF(self);
+        return NULL;
+    }
+    return (PyObject *) self;
+}
+
+static int
+picklebuf_traverse(PyPickleBufferObject *self, visitproc visit, void *arg)
+{
+    Py_VISIT(self->view.obj);
+    return 0;
+}
+
+static int
+picklebuf_clear(PyPickleBufferObject *self)
+{
+    PyBuffer_Release(&self->view);
+    return 0;
+}
+
+static void
+picklebuf_dealloc(PyPickleBufferObject *self)
+{
+    PyObject_GC_UnTrack(self);
+    if (self->weakreflist != NULL)
+        PyObject_ClearWeakRefs((PyObject *) self);
+    PyBuffer_Release(&self->view);
+    Py_TYPE(self)->tp_free((PyObject *) self);
+}
+
+/* Buffer API */
+
+static int
+picklebuf_getbuf(PyPickleBufferObject *self, Py_buffer *view, int flags)
+{
+    if (self->view.obj == NULL) {
+        PyErr_SetString(PyExc_ValueError,
+                        "operation forbidden on released PickleBuffer object");
+        return -1;
+    }
+    return PyObject_GetBuffer(self->view.obj, view, flags);
+}
+
+static void
+picklebuf_releasebuf(PyPickleBufferObject *self, Py_buffer *view)
+{
+    /* Since our bf_getbuffer redirects to the original object, this
+     * implementation is never called.  It only exists to signal that
+     * buffers exported by PickleBuffer have non-trivial releasing
+     * behaviour (see check in Python/getargs.c).
+     */
+}
+
+static PyBufferProcs picklebuf_as_buffer = {
+    .bf_getbuffer = (getbufferproc) picklebuf_getbuf,
+    .bf_releasebuffer = (releasebufferproc) picklebuf_releasebuf,
+};
+
+/* Methods */
+
+static PyObject *
+picklebuf_raw(PyPickleBufferObject *self, PyObject *Py_UNUSED(ignored))
+{
+    if (self->view.obj == NULL) {
+        PyErr_SetString(PyExc_ValueError,
+                        "operation forbidden on released PickleBuffer object");
+        return NULL;
+    }
+    if (self->view.suboffsets != NULL
+        || !PyBuffer_IsContiguous(&self->view, 'A')) {
+        PyErr_SetString(PyExc_BufferError,
+                        "cannot extract raw buffer from non-contiguous buffer");
+        return NULL;
+    }
+    PyObject *m = PyMemoryView_FromObject((PyObject *) self);
+    if (m == NULL) {
+        return NULL;
+    }
+    PyMemoryViewObject *mv = (PyMemoryViewObject *) m;
+    assert(mv->view.suboffsets == NULL);
+    /* Mutate memoryview instance to make it a "raw" memoryview */
+    mv->view.format = "B";
+    mv->view.ndim = 1;
+    mv->view.itemsize = 1;
+    /* shape = (length,) */
+    mv->view.shape = &mv->view.len;
+    /* strides = (1,) */
+    mv->view.strides = &mv->view.itemsize;
+    /* Fix memoryview state flags */
+    /* XXX Expose memoryobject.c's init_flags() instead? */
+    mv->flags = _Py_MEMORYVIEW_C | _Py_MEMORYVIEW_FORTRAN;
+    return m;
+}
+
+PyDoc_STRVAR(picklebuf_raw_doc,
+"raw($self, /)\n--\n\
+\n\
+Return a memoryview of the raw memory underlying this buffer.\n\
+Will raise BufferError is the buffer isn't contiguous.");
+
+static PyObject *
+picklebuf_release(PyPickleBufferObject *self, PyObject *Py_UNUSED(ignored))
+{
+    PyBuffer_Release(&self->view);
+    Py_RETURN_NONE;
+}
+
+PyDoc_STRVAR(picklebuf_release_doc,
+"release($self, /)\n--\n\
+\n\
+Release the underlying buffer exposed by the PickleBuffer object.");
+
+static PyMethodDef picklebuf_methods[] = {
+    {"raw",     (PyCFunction) picklebuf_raw,     METH_NOARGS, picklebuf_raw_doc},
+    {"release", (PyCFunction) picklebuf_release, METH_NOARGS, picklebuf_release_doc},
+    {NULL,      NULL}
+};
+
+PyTypeObject PyPickleBuffer_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    .tp_name = "pickle.PickleBuffer",
+    .tp_doc = "Wrapper for potentially out-of-band buffers",
+    .tp_basicsize = sizeof(PyPickleBufferObject),
+    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,
+    .tp_new = picklebuf_new,
+    .tp_dealloc = (destructor) picklebuf_dealloc,
+    .tp_traverse = (traverseproc) picklebuf_traverse,
+    .tp_clear = (inquiry) picklebuf_clear,
+    .tp_weaklistoffset = offsetof(PyPickleBufferObject, weakreflist),
+    .tp_as_buffer = &picklebuf_as_buffer,
+    .tp_methods = picklebuf_methods,
+};

PCbuild/pythoncore.vcxproj

@@ -197,6 +197,7 @@
     <ClInclude Include="..\Include\osmodule.h" />
     <ClInclude Include="..\Include\parsetok.h" />
     <ClInclude Include="..\Include\patchlevel.h" />
+    <ClInclude Include="..\Include\picklebufobject.h" />
     <ClInclude Include="..\Include\pyhash.h" />
     <ClInclude Include="..\Include\py_curses.h" />
     <ClInclude Include="..\Include\pyarena.h" />
@@ -383,6 +384,7 @@
     <ClCompile Include="..\Objects\object.c" />
     <ClCompile Include="..\Objects\obmalloc.c" />
     <ClCompile Include="..\Objects\odictobject.c" />
+    <ClCompile Include="..\Objects\picklebufobject.c" />
     <ClCompile Include="..\Objects\rangeobject.c" />
     <ClCompile Include="..\Objects\setobject.c" />
     <ClCompile Include="..\Objects\sliceobject.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -285,6 +285,9 @@
     <ClInclude Include="..\Include\patchlevel.h">
       <Filter>Include</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\picklebufobject.h">
+      <Filter>Include</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\py_curses.h">
       <Filter>Include</Filter>
     </ClInclude>
@@ -818,6 +821,9 @@
     <ClCompile Include="..\Objects\obmalloc.c">
       <Filter>Objects</Filter>
     </ClCompile>
+    <ClCompile Include="..\Objects\picklebufobject.c">
+      <Filter>Objects</Filter>
+    </ClCompile>
     <ClCompile Include="..\Objects\rangeobject.c">
       <Filter>Objects</Filter>
     </ClCompile>