[Scipy-svn] r6764 - in trunk/scipy/io/matlab: . tests

scipy-svn@scip... scipy-svn@scip...
Sat Sep 11 20:01:32 CDT 2010


Author: ptvirtan
Date: 2010-09-11 20:01:32 -0500 (Sat, 11 Sep 2010)
New Revision: 6764

Added:
   trunk/scipy/io/matlab/py3k.h
Modified:
   trunk/scipy/io/matlab/pyalloc.pxd
   trunk/scipy/io/matlab/streams.pyx
   trunk/scipy/io/matlab/tests/test_streams.py
Log:
3K: io/matlab: use only GenericStream on Py3 -- getting anything else to work reliably seems hopeless

Added: trunk/scipy/io/matlab/py3k.h
===================================================================
--- trunk/scipy/io/matlab/py3k.h	                        (rev 0)
+++ trunk/scipy/io/matlab/py3k.h	2010-09-12 01:01:32 UTC (rev 6764)
@@ -0,0 +1,104 @@
+#include <Python.h>
+#include <stdio.h>
+
+#if PY_VERSION_HEX < 0x03000000
+
+#include "cStringIO.h"
+
+#define npy_PyFile_Dup(file, mode) PyFile_AsFile(file)
+#define npy_PyFile_DupClose(file, handle) (0)
+#define npy_PyFile_Check PyFile_Check
+
+#else
+
+/*
+ * No-op implementation -- always fall back to the generic one.
+ */
+
+static struct PycStringIO_CAPI {
+    int(*cread)(PyObject *, char **, Py_ssize_t);
+    int(*creadline)(PyObject *, char **);
+    int(*cwrite)(PyObject *, const char *, Py_ssize_t);
+    PyObject *(*cgetvalue)(PyObject *);
+    PyObject *(*NewOutput)(int);
+    PyObject *(*NewInput)(PyObject *);
+    PyTypeObject *InputType, *OutputType;
+} *PycStringIO;
+
+static void PycString_IMPORT() {}
+
+#define PycStringIO_InputCheck(O) 0
+#define PycStringIO_OutputCheck(O) 0
+
+/*
+ * PyFile_* compatibility
+ */
+
+/*
+ * Get a FILE* handle to the file represented by the Python object
+ */
+static FILE*
+npy_PyFile_Dup(PyObject *file, char *mode)
+{
+    int fd, fd2;
+    PyObject *ret, *os;
+    /* Flush first to ensure things end up in the file in the correct order */
+    ret = PyObject_CallMethod(file, "flush", "");
+    if (ret == NULL) {
+        return NULL;
+    }
+    Py_DECREF(ret);
+    fd = PyObject_AsFileDescriptor(file);
+    if (fd == -1) {
+        return NULL;
+    }
+    os = PyImport_ImportModule("os");
+    if (os == NULL) {
+        return NULL;
+    }
+    ret = PyObject_CallMethod(os, "dup", "i", fd);
+    Py_DECREF(os);
+    if (ret == NULL) {
+        return NULL;
+    }
+    fd2 = PyNumber_AsSsize_t(ret, NULL);
+    Py_DECREF(ret);
+#ifdef _WIN32
+    return _fdopen(fd2, mode);
+#else
+    return fdopen(fd2, mode);
+#endif
+}
+
+/*
+ * Close the dup-ed file handle, and seek the Python one to the current position
+ */
+static int
+npy_PyFile_DupClose(PyObject *file, FILE* handle)
+{
+    PyObject *ret;
+    long position;
+    position = ftell(handle);
+    fclose(handle);
+
+    ret = PyObject_CallMethod(file, "seek", "li", position, 0);
+    if (ret == NULL) {
+        return -1;
+    }
+    Py_DECREF(ret);
+    return 0;
+}
+
+static int
+npy_PyFile_Check(PyObject *file)
+{
+    int fd;
+    fd = PyObject_AsFileDescriptor(file);
+    if (fd == -1) {
+        PyErr_Clear();
+        return 0;
+    }
+    return 1;
+}
+
+#endif

Modified: trunk/scipy/io/matlab/pyalloc.pxd
===================================================================
--- trunk/scipy/io/matlab/pyalloc.pxd	2010-09-12 01:01:14 UTC (rev 6763)
+++ trunk/scipy/io/matlab/pyalloc.pxd	2010-09-12 01:01:32 UTC (rev 6764)
@@ -1,13 +1,13 @@
 # -*- python -*- or rather like
 
-from python_string cimport PyString_FromStringAndSize, \
-    PyString_AS_STRING, PyString_Size
+from cpython cimport PyBytes_FromStringAndSize, \
+    PyBytes_AS_STRING, PyBytes_Size
 
 
 # Function to allocate, wrap memory via Python string creation
 cdef inline object pyalloc_v(Py_ssize_t n, void **pp):
-    cdef object ob = PyString_FromStringAndSize(NULL, n)
-    pp[0] = <void*> PyString_AS_STRING(ob)
+    cdef object ob = PyBytes_FromStringAndSize(NULL, n)
+    pp[0] = <void*> PyBytes_AS_STRING(ob)
     return ob
 
 

Modified: trunk/scipy/io/matlab/streams.pyx
===================================================================
--- trunk/scipy/io/matlab/streams.pyx	2010-09-12 01:01:14 UTC (rev 6763)
+++ trunk/scipy/io/matlab/streams.pyx	2010-09-12 01:01:32 UTC (rev 6764)
@@ -1,8 +1,10 @@
 # -*- python -*- or near enough
 
-from python_string cimport PyString_FromStringAndSize, \
-    PyString_AS_STRING, PyString_Size
+import sys
 
+from cpython cimport PyBytes_FromStringAndSize, \
+    PyBytes_AS_STRING, PyBytes_Size
+
 from pyalloc cimport pyalloc_v
 
 cdef extern from "stdlib.h" nogil:
@@ -17,28 +19,26 @@
     ctypedef struct PyObject:
         pass
     ctypedef struct FILE
-    FILE *PyFile_AsFile(object)
     size_t fread (void *ptr, size_t size, size_t n, FILE* fptr)
     int fseek (FILE * fptr, long int offset, int whence)
     long int ftell (FILE *stream)
 
-cdef extern from "fileobject.h":
-    ctypedef class __builtin__.file [object PyFileObject]:
-        pass
-
-cdef extern from "cStringIO.h":
+cdef extern from "py3k.h":
     # From:
     # http://svn.pyamf.org/pyamf/tags/release-0.4rc2/cpyamf/util.pyx
     # (MIT license) - with thanks
     void PycString_IMPORT()
-    object StringIO_NewOutput "PycStringIO->NewOutput" (int)
-    object StringIO_NewInput "PycStringIO->NewInput" (object)
     int StringIO_cread "PycStringIO->cread" (object, char **, Py_ssize_t)
     int StringIO_creadline "PycStringIO->creadline" (object, char **)
     int StringIO_cwrite "PycStringIO->cwrite" (object, char *, Py_ssize_t)
     object StringIO_cgetvalue "PycStringIO->cgetvalue" (obj)
     bint PycStringIO_InputCheck(object O)
     bint PycStringIO_OutputCheck(object O)
+
+    FILE* npy_PyFile_Dup(object file, char *mode) except NULL
+    int npy_PyFile_DupClose(object file, FILE *handle) except -1
+    int npy_PyFile_Check(object file)
+
        
 # initialize cStringIO
 PycString_IMPORT
@@ -60,11 +60,11 @@
         return self.fobj.read(n_bytes)
 
     cdef int read_into(self, void *buf, size_t n) except -1:
-        ''' Read n bytes from stream into pre-allocated buffer `buf`
-        '''
+        """ Read n bytes from stream into pre-allocated buffer `buf`
+        """
         cdef char* d_ptr
         data = self.fobj.read(n)
-        if PyString_Size(data) != n:
+        if PyBytes_Size(data) != n:
             raise IOError('could not read bytes')
             return -1
         d_ptr = data
@@ -72,15 +72,15 @@
         return 0
 
     cdef object read_string(self, size_t n, void **pp, int copy=True):
-        ''' Make new memory, wrap with object '''
+        """ Make new memory, wrap with object """
         data = self.fobj.read(n)
-        if PyString_Size(data) != n:
+        if PyBytes_Size(data) != n:
             raise IOError('could not read bytes')
         if copy != True:
-           pp[0] = <void*>PyString_AS_STRING(data)
+           pp[0] = <void*>PyBytes_AS_STRING(data)
            return data
         cdef object d_copy = pyalloc_v(n, pp)
-        memcpy(pp[0], PyString_AS_STRING(data), n)
+        memcpy(pp[0], PyBytes_AS_STRING(data), n)
         return d_copy
 
 
@@ -95,8 +95,8 @@
             return GenericStream.seek(self, offset, whence)
 
     cdef int read_into(self, void *buf, size_t n) except -1:
-        ''' Read n bytes from stream into pre-allocated buffer `buf`
-        '''
+        """ Read n bytes from stream into pre-allocated buffer `buf`
+        """
         cdef:
             size_t n_red
             char* d_ptr
@@ -107,10 +107,10 @@
         return 0
 
     cdef object read_string(self, size_t n, void **pp, int copy=True):
-        ''' Make new memory, wrap with object
+        """ Make new memory, wrap with object
 
         It's not obvious to me how to avoid a copy
-        '''
+        """
         cdef:
             char *d_ptr
             object obj
@@ -127,11 +127,14 @@
 
     def __init__(self, fobj):
         self.fobj = fobj
-        self.file = PyFile_AsFile(fobj)
-        
+        self.file = npy_PyFile_Dup(fobj, "rb")
+
+    def __del__(self):
+        npy_PyFile_DupClose(self.fobj, self.file)
+
     cpdef int seek(self, long int offset, int whence=0) except -1:
         cdef int ret
-        ''' move `offset` bytes in stream
+        """ move `offset` bytes in stream
 
         Parameters
         ----------
@@ -148,7 +151,7 @@
         Returns
         -------
         ret : int
-        '''
+        """
         ret = fseek(self.file, offset, whence)
         if ret:
             raise IOError('Failed seek')
@@ -159,8 +162,8 @@
         return ftell(self.file)
 
     cdef int read_into(self, void *buf, size_t n) except -1:
-        ''' Read n bytes from stream into pre-allocated buffer `buf`
-        '''
+        """ Read n bytes from stream into pre-allocated buffer `buf`
+        """
         cdef:
             size_t n_red
             char* d_ptr
@@ -171,18 +174,17 @@
         return 0
 
     cdef object read_string(self, size_t n, void **pp, int copy=True):
-        ''' Make new memory, wrap with object '''
+        """ Make new memory, wrap with object """
         cdef object obj = pyalloc_v(n, pp)
         cdef size_t n_red = fread(pp[0], 1, n, self.file)
         if n_red != n:
             raise IOError('could not read bytes')
         return obj
 
-
 def _read_into(GenericStream st, size_t n):
     # for testing only.  Use st.read instead
     cdef char * d_ptr
-    my_str = ' ' * n
+    my_str = b' ' * n
     d_ptr = my_str
     st.read_into(d_ptr, n)
     return my_str
@@ -192,17 +194,20 @@
     # for testing only.  Use st.read instead
     cdef char *d_ptr
     cdef object obj = st.read_string(n, <void **>&d_ptr, True)
-    my_str = 'A' * n
+    my_str = b'A' * n
     cdef char *mys_ptr = my_str
     memcpy(mys_ptr, d_ptr, n)
     return my_str
 
     
 cpdef GenericStream make_stream(object fobj):
-    ''' Make stream of correct type for file-like `fobj`
-    '''
-    if isinstance(fobj, file):
-        return FileStream(fobj)
+    """ Make stream of correct type for file-like `fobj`
+    """
+    if npy_PyFile_Check(fobj):
+        if sys.version_info[0] >= 3:
+            return GenericStream(fobj)
+        else:
+            return FileStream(fobj)
     elif PycStringIO_InputCheck(fobj) or PycStringIO_OutputCheck(fobj):
         return cStringStream(fobj)
     return GenericStream(fobj)

Modified: trunk/scipy/io/matlab/tests/test_streams.py
===================================================================
--- trunk/scipy/io/matlab/tests/test_streams.py	2010-09-12 01:01:14 UTC (rev 6763)
+++ trunk/scipy/io/matlab/tests/test_streams.py	2010-09-12 01:01:32 UTC (rev 6764)
@@ -4,11 +4,19 @@
 
 import os
 
-import StringIO
-import cStringIO
+import sys
+
+if sys.version_info[0] >= 3:
+    from io import BytesIO
+    cStringIO = BytesIO
+else:
+    from cStringIO import StringIO as cStringIO
+    from StringIO import StringIO as BytesIO
+
 from tempfile import mkstemp
 
 import numpy as np
+from numpy.compat import asbytes
 
 from nose.tools import assert_true, assert_false, \
      assert_equal, assert_raises
@@ -22,15 +30,15 @@
 
 
 def setup():
-    val = 'a\x00string'
+    val = asbytes('a\x00string')
     global fs, gs, cs, fname
     fd, fname = mkstemp()
     fs = os.fdopen(fd, 'wb')
     fs.write(val)
     fs.close()
-    fs = open(fname)
-    gs = StringIO.StringIO(val)
-    cs = cStringIO.StringIO(val)
+    fs = open(fname, 'rb')
+    gs = BytesIO(val)
+    cs = cStringIO(val)
 
 
 def teardown():
@@ -42,9 +50,10 @@
 def test_make_stream():
     global fs, gs, cs
     # test stream initialization
-    yield assert_true, isinstance(make_stream(gs), GenericStream)
-    yield assert_true, isinstance(make_stream(cs), cStringStream)
-    yield assert_true, isinstance(make_stream(fs), FileStream)
+    assert_true(isinstance(make_stream(gs), GenericStream))
+    if sys.version_info[0] < 3:
+        assert_true(isinstance(make_stream(cs), cStringStream))
+        assert_true(isinstance(make_stream(fs), FileStream))
 
 
 def test_tell_seek():
@@ -71,23 +80,23 @@
         st = make_stream(s)
         st.seek(0)
         res = st.read(-1)
-        yield assert_equal, res, 'a\x00string'
+        yield assert_equal, res, asbytes('a\x00string')
         st.seek(0)
         res = st.read(4)
-        yield assert_equal, res, 'a\x00st'
+        yield assert_equal, res, asbytes('a\x00st')
         # read into
         st.seek(0)
         res = _read_into(st, 4)
-        yield assert_equal, res, 'a\x00st'
+        yield assert_equal, res, asbytes('a\x00st')
         res = _read_into(st, 4)
-        yield assert_equal, res, 'ring'
+        yield assert_equal, res, asbytes('ring')
         yield assert_raises, IOError, _read_into, st, 2
         # read alloc
         st.seek(0)
         res = _read_string(st, 4)
-        yield assert_equal, res, 'a\x00st'
+        yield assert_equal, res, asbytes('a\x00st')
         res = _read_string(st, 4)
-        yield assert_equal, res, 'ring'
+        yield assert_equal, res, asbytes('ring')
         yield assert_raises, IOError, _read_string, st, 2
 
 if __name__ == "__main__":



More information about the Scipy-svn mailing list