[Scipy-svn] r5582 - in trunk/scipy/io/matlab: . tests

scipy-svn@scip... scipy-svn@scip...
Sat Feb 21 04:12:09 CST 2009


Author: matthew.brett@gmail.com
Date: 2009-02-21 04:12:05 -0600 (Sat, 21 Feb 2009)
New Revision: 5582

Modified:
   trunk/scipy/io/matlab/mio5.py
   trunk/scipy/io/matlab/miobase.py
   trunk/scipy/io/matlab/tests/test_mio.py
   trunk/scipy/io/matlab/zlibstreams.py
Log:
Avoid reading matlab functions (return error string); remove unpopular and slow zlib stream reader from use in reader

Modified: trunk/scipy/io/matlab/mio5.py
===================================================================
--- trunk/scipy/io/matlab/mio5.py	2009-02-20 23:40:19 UTC (rev 5581)
+++ trunk/scipy/io/matlab/mio5.py	2009-02-21 10:12:05 UTC (rev 5582)
@@ -24,7 +24,8 @@
 import scipy.sparse
 
 from miobase import MatFileReader, MatArrayReader, MatMatrixGetter, \
-     MatFileWriter, MatStreamWriter, docfiller, matdims
+     MatFileWriter, MatStreamWriter, docfiller, matdims, \
+     MatReadError
 
 miINT8 = 1
 miUINT8 = 2
@@ -65,9 +66,31 @@
 mxUINT64_CLASS = 15
 mxFUNCTION_CLASS = 16
 # Not doing anything with these at the moment.
-mxOPAQUE_CLASS = 17
+mxOPAQUE_CLASS = 17 # This appears to be a function workspace
+# https://www-old.cae.wisc.edu/pipermail/octave-maintainers/2007-May/002824.html
 mxOBJECT_CLASS_FROM_MATRIX_H = 18
 
+mxmap = { # Sometimes good for debug prints
+    mxCELL_CLASS: 'mxCELL_CLASS',
+    mxSTRUCT_CLASS: 'mxSTRUCT_CLASS',
+    mxOBJECT_CLASS: 'mxOBJECT_CLASS',
+    mxCHAR_CLASS: 'mxCHAR_CLASS',
+    mxSPARSE_CLASS: 'mxSPARSE_CLASS',
+    mxDOUBLE_CLASS: 'mxDOUBLE_CLASS',
+    mxSINGLE_CLASS: 'mxSINGLE_CLASS',
+    mxINT8_CLASS: 'mxINT8_CLASS',
+    mxUINT8_CLASS: 'mxUINT8_CLASS',
+    mxINT16_CLASS: 'mxINT16_CLASS',
+    mxUINT16_CLASS: 'mxUINT16_CLASS',
+    mxINT32_CLASS: 'mxINT32_CLASS',
+    mxUINT32_CLASS: 'mxUINT32_CLASS',
+    mxINT64_CLASS: 'mxINT64_CLASS',
+    mxUINT64_CLASS: 'mxUINT64_CLASS',
+    mxFUNCTION_CLASS: 'mxFUNCTION_CLASS',
+    mxOPAQUE_CLASS: 'mxOPAQUE_CLASS',
+    mxOBJECT_CLASS_FROM_MATRIX_H: 'mxOBJECT_CLASS_FROM_MATRIX_H',
+}
+
 mdtypes_template = {
     miINT8: 'i1',
     miUINT8: 'u1',
@@ -319,7 +342,7 @@
         if mc == mxOBJECT_CLASS:
             return Mat5ObjectMatrixGetter(self, header)
         if mc == mxFUNCTION_CLASS:
-            return Mat5FunctionMatrixGetter(self, header)
+            return Mat5FunctionGetter(self, header)
         raise TypeError, 'No reader for class code %s' % mc
 
 
@@ -329,25 +352,11 @@
     Sets up reader for gzipped stream on init, providing wrapper
     for this new sub-stream.
 
-    Note that we use a zlib stream reader to return the data from the
-    zlib compressed stream.
-
-    In our case, we want this reader (under the hood) to do one small
-    read of the stream to get enough data to check the variable name,
-    because we may want to skip this variable - for which we need to
-    check the name.  If we need to read the rest of the data (not
-    skipping), then (under the hood) the steam reader decompresses the
-    whole of the rest of the stream ready for returning here to
-    construct the array.  This avoids the overhead of reading the
-    stream in small chunks - the default behavior of our zlib stream
-    reader.
-
-    This is why we use TwoShotZlibInputStream below.
     '''
     def __init__(self, array_reader, byte_count):
+        instr = array_reader.mat_stream.read(byte_count)
         super(Mat5ZArrayReader, self).__init__(
-            TwoShotZlibInputStream(array_reader.mat_stream,
-                                   byte_count),
+            StringIO(zlib.decompress(instr)),
             array_reader.dtypes,
             array_reader.processor_func,
             array_reader.codecs,
@@ -516,10 +525,9 @@
         return MatlabObject(result, classname)
 
 
-class Mat5FunctionMatrixGetter(Mat5CellMatrixGetter):
+class Mat5FunctionGetter(Mat5ObjectMatrixGetter):
     def get_raw_array(self):
-        result = super(Mat5FunctionMatrixGetter, self).get_raw_array()
-        return MatlabFunction(result)
+        raise MatReadError('Cannot read matlab functions')
 
 
 class MatFile5Reader(MatFileReader):

Modified: trunk/scipy/io/matlab/miobase.py
===================================================================
--- trunk/scipy/io/matlab/miobase.py	2009-02-20 23:40:19 UTC (rev 5581)
+++ trunk/scipy/io/matlab/miobase.py	2009-02-21 10:12:05 UTC (rev 5582)
@@ -10,6 +10,7 @@
 
 import byteordercodes as boc
 
+class MatReadError(Exception): pass
 
 doc_dict = \
     {'file_arg':
@@ -136,6 +137,7 @@
     else:
         raise ValueError('Unknown mat file type, version %s' % ret)
 
+class MatReadError(Exception): pass
 
 def matdims(arr, oned_as='column'):
     ''' Determine equivalent matlab dimensions for given array 
@@ -412,7 +414,7 @@
                 arr = np.squeeze(arr)
                 if not arr.size:
                     arr = np.array([])
-                elif not arr.shape: # 0d coverted to scalar
+                elif not arr.shape and arr.dtype.isbuiltin: # 0d coverted to scalar
                     arr = arr.item()
             return arr
         return func
@@ -442,7 +444,11 @@
             if variable_names and name not in variable_names:
                 getter.to_next()
                 continue
-            res = getter.get_array()
+            try:
+                res = getter.get_array()
+            except MatReadError, err:
+                res = "Read error: %s" % err
+                getter.to_next()
             mdict[name] = res
             if getter.is_global:
                 mdict['__globals__'].append(name)

Modified: trunk/scipy/io/matlab/tests/test_mio.py
===================================================================
--- trunk/scipy/io/matlab/tests/test_mio.py	2009-02-20 23:40:19 UTC (rev 5581)
+++ trunk/scipy/io/matlab/tests/test_mio.py	2009-02-21 10:12:05 UTC (rev 5582)
@@ -188,16 +188,14 @@
     {'name': 'sparsecomplex',
      'expected': {'testsparsecomplex': SP.coo_matrix(B)},
      })
-'''
+# We cannot read matlab functions for the moment
 case_table5.append(
     {'name': 'func',
-     'expected': {'testfunc': 0},
+     'expected': {'testfunc': 'Read error: Cannot read matlab functions'},
      })
-'''
 
-# These should also have matlab load equivalents,
-case_table5_rt = case_table5[:]
-# Inline functions can't be concatenated in matlab
+case_table5_rt = case_table5[:-1] # not the function read write
+# Inline functions can't be concatenated in matlab, so RT only
 case_table5_rt.append(
     {'name': 'objectarray',
      'expected': {'testobjectarray': np.repeat(MO, 2).reshape(1,2)}})

Modified: trunk/scipy/io/matlab/zlibstreams.py
===================================================================
--- trunk/scipy/io/matlab/zlibstreams.py	2009-02-20 23:40:19 UTC (rev 5581)
+++ trunk/scipy/io/matlab/zlibstreams.py	2009-02-21 10:12:05 UTC (rev 5582)
@@ -122,13 +122,14 @@
             return
         # read until we have enough bytes in the buffer
         read_to_end = bytes == -1
-
+        
+        bytes_to_fill = bytes - len(self.data)
+        if not (bytes_to_fill or read_to_end):
+            return
         # store data chunks in a list until the end so that we avoid the
         # quadratic behavior of continuously extending a string
         data_chunks = [self.data]
-        bytes_to_fill = bytes - len(self.data)
-
-        while read_to_end or bytes_to_fill > 0:
+        while bytes_to_fill > 0 or read_to_end:
             z_n_to_fetch = self._blocksize_iterator.next()
             if z_n_to_fetch == 0:
                 self.exhausted = True
@@ -198,11 +199,13 @@
             string containing read data
 
         '''
-        self.__fill(bytes)
         if bytes == -1:
+            self.__fill(bytes)
             data = self.data
             self.data = ""
         else:
+            if len(self.data) < bytes:
+                self.__fill(bytes)
             data = self.data[:bytes]
             self.data = self.data[bytes:]
         self.unzipped_pos += len(data)



More information about the Scipy-svn mailing list