[Numpy-svn] r8152 - in trunk/numpy/core: . src/multiarray tests

numpy-svn@scip... numpy-svn@scip...
Sat Feb 20 12:09:46 CST 2010


Author: ptvirtan
Date: 2010-02-20 12:09:45 -0600 (Sat, 20 Feb 2010)
New Revision: 8152

Added:
   trunk/numpy/core/src/multiarray/numpymemoryview.c
   trunk/numpy/core/src/multiarray/numpymemoryview.h
Modified:
   trunk/numpy/core/SConscript
   trunk/numpy/core/setup.py
   trunk/numpy/core/src/multiarray/arrayobject.c
   trunk/numpy/core/src/multiarray/buffer.c
   trunk/numpy/core/src/multiarray/common.c
   trunk/numpy/core/src/multiarray/ctors.c
   trunk/numpy/core/src/multiarray/multiarraymodule.c
   trunk/numpy/core/tests/test_multiarray.py
Log:
ENH: implement PEP 3118 buffer consumer on Py2.6, adding a simple Memoryview object

To use PEP 3118 buffers as ndarray bases, we need to have a way to
release the buffers on garbage collection. The easiest way to do this is
to use a Memoryview object -- but they are not available on Python 2.6.

Hence, we implement a minimal memory view object that only handles the
garbage collection, and use it on Python 2.6.  It also implements the
new buffer protocol, to allow testing the functionality.

Modified: trunk/numpy/core/SConscript
===================================================================
--- trunk/numpy/core/SConscript	2010-02-20 18:09:17 UTC (rev 8151)
+++ trunk/numpy/core/SConscript	2010-02-20 18:09:45 UTC (rev 8152)
@@ -447,6 +447,7 @@
         pjoin('src', 'multiarray', 'conversion_utils.c'),
         pjoin('src', 'multiarray', 'usertypes.c'),
         pjoin('src', 'multiarray', 'buffer.c'),
+        pjoin('src', 'multiarray', 'numpymemoryview.c'),
         pjoin('src', 'multiarray', 'scalarapi.c')]
     multiarray_src.extend(arraytypes_src)
     multiarray_src.extend(scalartypes_src)

Modified: trunk/numpy/core/setup.py
===================================================================
--- trunk/numpy/core/setup.py	2010-02-20 18:09:17 UTC (rev 8151)
+++ trunk/numpy/core/setup.py	2010-02-20 18:09:45 UTC (rev 8152)
@@ -703,6 +703,7 @@
             join('src', 'multiarray', 'mapping.h'),
             join('src', 'multiarray', 'methods.h'),
             join('src', 'multiarray', 'multiarraymodule.h'),
+            join('src', 'multiarray', 'numpymemoryview.h'),
             join('src', 'multiarray', 'number.h'),
             join('src', 'multiarray', 'numpyos.h'),
             join('src', 'multiarray', 'refcount.h'),
@@ -715,6 +716,7 @@
     multiarray_src = [join('src', 'multiarray', 'multiarraymodule.c'),
         join('src', 'multiarray', 'hashdescr.c'),
         join('src', 'multiarray', 'arrayobject.c'),
+        join('src', 'multiarray', 'numpymemoryview.c'),
         join('src', 'multiarray', 'buffer.c'),
         join('src', 'multiarray', 'datetime.c'),
         join('src', 'multiarray', 'numpyos.c'),

Modified: trunk/numpy/core/src/multiarray/arrayobject.c
===================================================================
--- trunk/numpy/core/src/multiarray/arrayobject.c	2010-02-20 18:09:17 UTC (rev 8151)
+++ trunk/numpy/core/src/multiarray/arrayobject.c	2010-02-20 18:09:45 UTC (rev 8152)
@@ -1317,6 +1317,9 @@
 #if !defined(NPY_PY3K)
      | Py_TPFLAGS_CHECKTYPES
 #endif
+#if (PY_VERSION_HEX >= 0x02060000) && (PY_VERSION_HEX < 0x03000000)
+     | Py_TPFLAGS_HAVE_NEWBUFFER
+#endif
      | Py_TPFLAGS_BASETYPE),                  /* tp_flags */
     0,                                          /* tp_doc */
 

Modified: trunk/numpy/core/src/multiarray/buffer.c
===================================================================
--- trunk/numpy/core/src/multiarray/buffer.c	2010-02-20 18:09:17 UTC (rev 8151)
+++ trunk/numpy/core/src/multiarray/buffer.c	2010-02-20 18:09:45 UTC (rev 8152)
@@ -636,8 +636,8 @@
         return NULL;
     }
     str = PyUString_FromStringAndSize(buf, strlen(buf));
-    descr = PyObject_CallMethod(_numpy_internal, "_dtype_from_pep3118",
-                                "O", str);
+    descr = (PyArray_Descr*)PyObject_CallMethod(
+        _numpy_internal, "_dtype_from_pep3118", "O", str);
     Py_DECREF(str);
     if (descr == NULL) {
         PyErr_Format(PyExc_ValueError,

Modified: trunk/numpy/core/src/multiarray/common.c
===================================================================
--- trunk/numpy/core/src/multiarray/common.c	2010-02-20 18:09:17 UTC (rev 8151)
+++ trunk/numpy/core/src/multiarray/common.c	2010-02-20 18:09:45 UTC (rev 8152)
@@ -153,7 +153,7 @@
     PyObject *ip;
     PyArray_Descr *chktype = NULL;
     PyArray_Descr *outtype;
-#if PY_VERSION_HEX >= 0x02070000
+#if PY_VERSION_HEX >= 0x02060000
     Py_buffer buffer_view;
 #endif
 
@@ -210,7 +210,7 @@
         goto finish;
     }
 
-#if PY_VERSION_HEX >= 0x02070000
+#if PY_VERSION_HEX >= 0x02060000
     /* PEP 3118 buffer interface */
     memset(&buffer_view, 0, sizeof(Py_buffer));
     if (PyObject_GetBuffer(op, &buffer_view, PyBUF_FORMAT|PyBUF_STRIDES) == 0 ||

Modified: trunk/numpy/core/src/multiarray/ctors.c
===================================================================
--- trunk/numpy/core/src/multiarray/ctors.c	2010-02-20 18:09:17 UTC (rev 8151)
+++ trunk/numpy/core/src/multiarray/ctors.c	2010-02-20 18:09:45 UTC (rev 8152)
@@ -17,6 +17,8 @@
 
 #include "ctors.h"
 
+#include "numpymemoryview.h"
+
 /*
  * Reading from a file or a string.
  *
@@ -1083,7 +1085,7 @@
 {
     int d = 0;
     PyObject *e;
-#if PY_VERSION_HEX >= 0x02070000
+#if PY_VERSION_HEX >= 0x02060000
     Py_buffer buffer_view;
 #endif
 
@@ -1117,7 +1119,7 @@
     if (stop_at_tuple && PyTuple_Check(s)) {
         return 0;
     }
-#if PY_VERSION_HEX >= 0x02070000
+#if PY_VERSION_HEX >= 0x02060000
     /* PEP 3118 buffer interface */
     memset(&buffer_view, 0, sizeof(Py_buffer));
     if (PyObject_GetBuffer(s, &buffer_view, PyBUF_STRIDES) == 0 ||
@@ -1631,12 +1633,7 @@
 NPY_NO_EXPORT int
 _array_from_buffer_3118(PyObject *obj, PyObject **out)
 {
-#if PY_VERSION_HEX >= 0x02070000
-    /* XXX: Pythons < 2.7 do not have the memory view object. This makes
-     *      using buffers somewhat difficult, since one cannot release them
-     *      using the garbage collection
-     */
-
+#if PY_VERSION_HEX >= 0x02060000
     /* PEP 3118 */
     PyObject *memoryview;
     Py_buffer *view;
@@ -1654,7 +1651,7 @@
 
     view = PyMemoryView_GET_BUFFER(memoryview);
     if (view->format != NULL) {
-        descr = _descriptor_from_pep3118_format(view->format);
+        descr = (PyObject*)_descriptor_from_pep3118_format(view->format);
         if (descr == NULL) {
             goto fail;
         }

Modified: trunk/numpy/core/src/multiarray/multiarraymodule.c
===================================================================
--- trunk/numpy/core/src/multiarray/multiarraymodule.c	2010-02-20 18:09:17 UTC (rev 8151)
+++ trunk/numpy/core/src/multiarray/multiarraymodule.c	2010-02-20 18:09:45 UTC (rev 8152)
@@ -41,6 +41,7 @@
 #include "calculation.h"
 #include "number.h"
 #include "scalartypes.h"
+#include "numpymemoryview.h"
 
 /*NUMPY_API
  * Get Priority from object
@@ -3051,6 +3052,14 @@
         goto err;
     }
 
+    /* Initialize types in numpymemoryview.c */
+    if (_numpymemoryview_init(&s) < 0) {
+        return RETVAL;
+    }
+    if (s != NULL) {
+        PyDict_SetItemString(d, "memorysimpleview", s);
+    }
+
     /*
      * PyExc_Exception should catch all the standard errors that are
      * now raised instead of the string exception "multiarray.error"

Added: trunk/numpy/core/src/multiarray/numpymemoryview.c
===================================================================
--- trunk/numpy/core/src/multiarray/numpymemoryview.c	                        (rev 0)
+++ trunk/numpy/core/src/multiarray/numpymemoryview.c	2010-02-20 18:09:45 UTC (rev 8152)
@@ -0,0 +1,310 @@
+/*
+ * Simple PyMemoryView'ish object for Python 2.6 compatibility.
+ *
+ * On Python >= 2.7, we can use the actual PyMemoryView objects.
+ *
+ * Some code copied from the CPython implementation.
+ */
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include "structmember.h"
+
+#define _MULTIARRAYMODULE
+#define NPY_NO_PREFIX
+#include "numpy/arrayobject.h"
+#include "numpy/arrayscalars.h"
+
+#include "npy_config.h"
+#include "npy_3kcompat.h"
+
+#include "numpymemoryview.h"
+
+
+#if (PY_VERSION_HEX >= 0x02060000) && (PY_VERSION_HEX < 0x02070000)
+
+/*
+ * Memory allocation
+ */
+
+static int
+memorysimpleview_traverse(PyMemorySimpleViewObject *self,
+                          visitproc visit, void *arg)
+{
+    if (self->base != NULL)
+        Py_VISIT(self->base);
+    if (self->view.obj != NULL)
+        Py_VISIT(self->view.obj);
+    return 0;
+}
+
+static int
+memorysimpleview_clear(PyMemorySimpleViewObject *self)
+{
+    Py_CLEAR(self->base);
+    PyBuffer_Release(&self->view);
+    self->view.obj = NULL;
+    return 0;
+}
+
+static void
+memorysimpleview_dealloc(PyMemorySimpleViewObject *self)
+{
+    PyObject_GC_UnTrack(self);
+    Py_CLEAR(self->base);
+    if (self->view.obj != NULL) {
+        PyBuffer_Release(&self->view);
+        self->view.obj = NULL;
+    }
+    PyObject_GC_Del(self);
+}
+
+static PyObject *
+memorysimpleview_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds)
+{
+    PyObject *obj;
+    static char *kwlist[] = {"object", 0};
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:memorysimpleview", kwlist,
+                                     &obj)) {
+        return NULL;
+    }
+    return PyMemorySimpleView_FromObject(obj);
+}
+
+
+/*
+ * Buffer interface
+ */
+
+static int
+memorysimpleview_getbuffer(PyMemorySimpleViewObject *self,
+                           Py_buffer *view, int flags)
+{
+    return PyObject_GetBuffer(self->base, view, flags);
+}
+
+static void
+memorysimpleview_releasebuffer(PyMemorySimpleViewObject *self,
+                               Py_buffer *view)
+{
+    PyBuffer_Release(view);
+}
+
+static PyBufferProcs memorysimpleview_as_buffer = {
+    (readbufferproc)0,       /*bf_getreadbuffer*/
+    (writebufferproc)0,     /*bf_getwritebuffer*/
+    (segcountproc)0,        /*bf_getsegcount*/
+    (charbufferproc)0,       /*bf_getcharbuffer*/
+    (getbufferproc)memorysimpleview_getbuffer, /* bf_getbuffer */
+    (releasebufferproc)memorysimpleview_releasebuffer, /* bf_releasebuffer */
+};
+
+
+/*
+ * Getters
+ */
+
+static PyObject *
+_IntTupleFromSsizet(int len, Py_ssize_t *vals)
+{
+    int i;
+    PyObject *o;
+    PyObject *intTuple;
+
+    if (vals == NULL) {
+        Py_INCREF(Py_None);
+        return Py_None;
+    }
+    intTuple = PyTuple_New(len);
+    if (!intTuple) return NULL;
+    for(i=0; i<len; i++) {
+        o = PyInt_FromSsize_t(vals[i]);
+        if (!o) {
+            Py_DECREF(intTuple);
+            return NULL;
+        }
+        PyTuple_SET_ITEM(intTuple, i, o);
+    }
+    return intTuple;
+}
+
+static PyObject *
+memorysimpleview_format_get(PyMemorySimpleViewObject *self)
+{
+    return PyUnicode_FromString(self->view.format);
+}
+
+static PyObject *
+memorysimpleview_itemsize_get(PyMemorySimpleViewObject *self)
+{
+    return PyLong_FromSsize_t(self->view.itemsize);
+}
+
+static PyObject *
+memorysimpleview_shape_get(PyMemorySimpleViewObject *self)
+{
+    return _IntTupleFromSsizet(self->view.ndim, self->view.shape);
+}
+
+static PyObject *
+memorysimpleview_strides_get(PyMemorySimpleViewObject *self)
+{
+    return _IntTupleFromSsizet(self->view.ndim, self->view.strides);
+}
+
+static PyObject *
+memorysimpleview_suboffsets_get(PyMemorySimpleViewObject *self)
+{
+    return _IntTupleFromSsizet(self->view.ndim, self->view.suboffsets);
+}
+
+static PyObject *
+memorysimpleview_readonly_get(PyMemorySimpleViewObject *self)
+{
+    return PyBool_FromLong(self->view.readonly);
+}
+
+static PyObject *
+memorysimpleview_ndim_get(PyMemorySimpleViewObject *self)
+{
+    return PyLong_FromLong(self->view.ndim);
+}
+
+
+static PyGetSetDef memorysimpleview_getsets[] =
+{
+    {"format", (getter)memorysimpleview_format_get, NULL, NULL, NULL},
+    {"itemsize", (getter)memorysimpleview_itemsize_get, NULL, NULL, NULL},
+    {"shape", (getter)memorysimpleview_shape_get, NULL, NULL, NULL},
+    {"strides", (getter)memorysimpleview_strides_get, NULL, NULL, NULL},
+    {"suboffsets", (getter)memorysimpleview_suboffsets_get, NULL, NULL, NULL},
+    {"readonly", (getter)memorysimpleview_readonly_get, NULL, NULL, NULL},
+    {"ndim", (getter)memorysimpleview_ndim_get, NULL, NULL, NULL},
+    {NULL, NULL, NULL, NULL}
+};
+
+NPY_NO_EXPORT PyTypeObject PyMemorySimpleView_Type = {
+#if defined(NPY_PY3K)
+    PyVarObject_HEAD_INIT(NULL, 0)
+#else
+    PyObject_HEAD_INIT(NULL)
+    0,                                          /* ob_size */
+#endif
+    "numpy.memorysimpleview",
+    sizeof(PyMemorySimpleViewObject),
+    0,                                          /* tp_itemsize */
+    /* methods */
+    (destructor)memorysimpleview_dealloc,       /* tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+#if defined(NPY_PY3K)
+    0,                                          /* tp_reserved */
+#else
+    (cmpfunc)0,                                 /* tp_compare */
+#endif
+    (reprfunc)0,                                /* tp_repr */
+    0,                                          /* tp_as_number */
+    0,                                          /* tp_as_sequence */
+    0,                                          /* tp_as_mapping */
+    0,                                          /* tp_hash */
+    0,                                          /* tp_call */
+    (reprfunc)0,                                /* tp_str */
+    0,                                          /* tp_getattro */
+    0,                                          /* tp_setattro */
+    &memorysimpleview_as_buffer,                /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
+    | Py_TPFLAGS_HAVE_NEWBUFFER,                /* tp_flags */
+    0,                                          /* tp_doc */
+    (traverseproc)memorysimpleview_traverse,    /* tp_traverse */
+    (inquiry)memorysimpleview_clear,            /* tp_clear */
+    0,                                          /* tp_richcompare */
+    0,                                          /* tp_weaklistoffset */
+    0,                                          /* tp_iter */
+    0,                                          /* tp_iternext */
+    0,                                          /* tp_methods */
+    0,                                          /* tp_members */
+    memorysimpleview_getsets,                   /* tp_getset */
+    0,                                          /* tp_base */
+    0,                                          /* tp_dict */
+    0,                                          /* tp_descr_get */
+    0,                                          /* tp_descr_set */
+    0,                                          /* tp_dictoffset */
+    0,                                          /* tp_init */
+    0,                                          /* tp_alloc */
+    memorysimpleview_new,                       /* tp_new */
+    0,                                          /* tp_free */
+    0,                                          /* tp_is_gc */
+    0,                                          /* tp_bases */
+    0,                                          /* tp_mro */
+    0,                                          /* tp_cache */
+    0,                                          /* tp_subclasses */
+    0,                                          /* tp_weaklist */
+    0,                                          /* tp_del */
+#if PY_VERSION_HEX >= 0x02060000
+    0,                                          /* tp_version_tag */
+#endif
+};
+
+
+/*
+ * Factory
+ */
+NPY_NO_EXPORT PyObject *
+PyMemorySimpleView_FromObject(PyObject *base)
+{
+    PyMemorySimpleViewObject *mview = NULL;
+    Py_buffer view;
+
+    if (Py_TYPE(base)->tp_as_buffer == NULL ||
+        Py_TYPE(base)->tp_as_buffer->bf_getbuffer == NULL) {
+
+        PyErr_SetString(PyExc_TypeError,
+            "cannot make memory view because object does "
+            "not have the buffer interface");
+        return NULL;
+    }
+
+    memset(&view, 0, sizeof(Py_buffer));
+    if (PyObject_GetBuffer(base, &view, PyBUF_FULL_RO) < 0)
+        return NULL;
+
+    mview = (PyMemorySimpleViewObject *)
+        PyObject_GC_New(PyMemorySimpleViewObject, &PyMemorySimpleView_Type);
+    if (mview == NULL) {
+        PyBuffer_Release(&view);
+        return NULL;
+    }
+    memcpy(&mview->view, &view, sizeof(Py_buffer));
+    mview->base = base;
+    Py_INCREF(base);
+
+    PyObject_GC_Track(mview);
+    return (PyObject *)mview;
+}
+
+
+/*
+ * Module initialization
+ */
+
+NPY_NO_EXPORT int
+_numpymemoryview_init(PyObject **typeobject)
+{
+    if (PyType_Ready(&PyMemorySimpleView_Type) < 0) {
+        return -1;
+    }
+    *typeobject = (PyObject*)&PyMemorySimpleView_Type;
+    return 0;
+}
+
+#else
+
+NPY_NO_EXPORT int
+_numpymemoryview_init(PyObject **typeobject)
+{
+    *typeobject = NULL;
+    return 0;
+}
+
+#endif


Property changes on: trunk/numpy/core/src/multiarray/numpymemoryview.c
___________________________________________________________________
Name: svn:keywords
   + Id Author
Name: svn:eol-style
   + native

Added: trunk/numpy/core/src/multiarray/numpymemoryview.h
===================================================================
--- trunk/numpy/core/src/multiarray/numpymemoryview.h	                        (rev 0)
+++ trunk/numpy/core/src/multiarray/numpymemoryview.h	2010-02-20 18:09:45 UTC (rev 8152)
@@ -0,0 +1,29 @@
+#ifndef _NPY_PRIVATE_NUMPYMEMORYVIEW_H_
+#define _NPY_PRIVATE_NUMPYMEMORYVIEW_H_
+
+/*
+ * Memoryview is introduced to 2.x series only in 2.7, so for supporting 2.6,
+ * we need to have a minimal implementation here.
+ */
+#if (PY_VERSION_HEX >= 0x02060000) && (PY_VERSION_HEX < 0x02070000)
+
+typedef struct {
+    PyObject_HEAD
+    PyObject *base;
+    Py_buffer view;
+} PyMemorySimpleViewObject;
+
+NPY_NO_EXPORT PyObject *
+PyMemorySimpleView_FromObject(PyObject *base);
+
+#define PyMemorySimpleView_GET_BUFFER(op) (&((PyMemorySimpleViewObject *)(op))->view)
+
+#define PyMemoryView_FromObject PyMemorySimpleView_FromObject
+#define PyMemoryView_GET_BUFFER PyMemorySimpleView_GET_BUFFER
+
+#endif
+
+NPY_NO_EXPORT int
+_numpymemoryview_init(PyObject **typeobject);
+
+#endif


Property changes on: trunk/numpy/core/src/multiarray/numpymemoryview.h
___________________________________________________________________
Name: svn:keywords
   + Id Author
Name: svn:eol-style
   + native

Modified: trunk/numpy/core/tests/test_multiarray.py
===================================================================
--- trunk/numpy/core/tests/test_multiarray.py	2010-02-20 18:09:17 UTC (rev 8151)
+++ trunk/numpy/core/tests/test_multiarray.py	2010-02-20 18:09:45 UTC (rev 8152)
@@ -1451,7 +1451,11 @@
         assert_raises(np.ComplexWarning, x.__setitem__, slice(None), y)
         warnings.simplefilter("default", np.ComplexWarning)
 
-if sys.version_info >= (2, 7):
+if sys.version_info >= (2, 6):
+
+    if sys.version_info[:2] == (2, 6):
+        from numpy.core.multiarray import memorysimpleview as memoryview
+
     class TestNewBufferProtocol(object):
         def _check_roundtrip(self, obj):
             obj = np.asarray(obj)
@@ -1556,9 +1560,9 @@
             x = np.array(([[1,2],[3,4]],), dtype=[('a', (int, (2,2)))])
             y = memoryview(x)
             assert y.format == 'T{(2,2)=l:a:}'
-            assert y.shape == ()
+            assert y.shape is None
             assert y.ndim == 0
-            assert y.strides == ()
+            assert y.strides is None
             assert y.suboffsets is None
             assert y.itemsize == 16
 



More information about the Numpy-svn mailing list