[Scipy-svn] r4775 - branches/mb_mio_refactor branches/mb_mio_refactor/matlab trunk/scipy/io/matlab/tests trunk/scipy/io/matlab/tests/data

scipy-svn@scip... scipy-svn@scip...
Sat Oct 4 23:49:49 CDT 2008


Author: matthew.brett@gmail.com
Date: 2008-10-04 23:49:23 -0500 (Sat, 04 Oct 2008)
New Revision: 4775

Added:
   branches/mb_mio_refactor/matlab/byteordercodes.py
   trunk/scipy/io/matlab/tests/data/testhdf5_7.4_GLNX86.mat
   trunk/scipy/io/matlab/tests/test_byteordercodes.py
Modified:
   branches/mb_mio_refactor/__init__.py
   branches/mb_mio_refactor/matlab/__init__.py
   branches/mb_mio_refactor/matlab/mio.py
   branches/mb_mio_refactor/matlab/mio4.py
   branches/mb_mio_refactor/matlab/mio5.py
   branches/mb_mio_refactor/matlab/miobase.py
   trunk/scipy/io/matlab/tests/save_test.m
   trunk/scipy/io/matlab/tests/test_mio.py
Log:
Fixing long-broken mio tests

Modified: branches/mb_mio_refactor/__init__.py
===================================================================
--- branches/mb_mio_refactor/__init__.py	2008-10-05 03:36:54 UTC (rev 4774)
+++ branches/mb_mio_refactor/__init__.py	2008-10-05 04:49:23 UTC (rev 4775)
@@ -80,9 +80,10 @@
 from netcdf import netcdf_file, netcdf_variable
 
 from recaster import sctype_attributes, Recaster
+import matlab.byteordercodes as byteordercodes
 from data_store import save_as_module
 from mmio import mminfo, mmread, mmwrite
 
 __all__ = filter(lambda s:not s.startswith('_'),dir())
-from scipy.testing.pkgtester import Tester
+from numpy.testing import Tester
 test = Tester().test

Modified: branches/mb_mio_refactor/matlab/__init__.py
===================================================================
--- branches/mb_mio_refactor/matlab/__init__.py	2008-10-05 03:36:54 UTC (rev 4774)
+++ branches/mb_mio_refactor/matlab/__init__.py	2008-10-05 04:49:23 UTC (rev 4775)
@@ -0,0 +1,5 @@
+# Matlab file read and write utilities
+from mio import loadmat, savemat
+
+from numpy.testing import Tester
+test = Tester().test

Added: branches/mb_mio_refactor/matlab/byteordercodes.py
===================================================================
--- branches/mb_mio_refactor/matlab/byteordercodes.py	2008-10-05 03:36:54 UTC (rev 4774)
+++ branches/mb_mio_refactor/matlab/byteordercodes.py	2008-10-05 04:49:23 UTC (rev 4775)
@@ -0,0 +1,68 @@
+''' Byteorder utilities for system - numpy byteorder encoding
+
+Converts a variety of string codes for little endian, big endian,
+native byte order and swapped byte order to explicit numpy endian
+codes - one of '<' (little endian) or '>' (big endian)
+
+'''
+
+import sys
+
+sys_is_le = sys.byteorder == 'little'
+native_code = sys_is_le and '<' or '>'
+swapped_code = sys_is_le and '>' or '<'
+
+aliases = {'little': ('little', '<', 'l', 'le'),
+           'big': ('big', '>', 'b', 'be'),
+           'native': ('native', '='),
+           'swapped': ('swapped', 'S')}
+
+def to_numpy_code(code):
+    ''' Convert various order codings to numpy format 
+    Parameters
+    ----------
+    code : {'little','big','l','b','le','be','<','>',
+             'native','=',
+             'swapped', 's'} string
+          code is converted to lower case before parsing
+    
+    Returns
+    -------
+    out_code : {'<','>'} string
+             where '<' is the numpy dtype code for little 
+             endian, and '>' is the code for big endian
+    
+
+    Examples
+    --------
+    >>> import sys
+    >>> from imagers.byteorder import to_numpy_code, sys_is_le
+    >>> sys_is_le == (sys.byteorder == 'little')
+    True
+    >>> to_numpy_code('big')
+    '>'
+    >>> to_numpy_code('little')
+    '<'
+    >>> nc = to_numpy_code('native')
+    >>> nc == '<' if sys_is_le else nc == '>'
+    True
+    >>> sc = to_numpy_code('swapped')
+    >>> sc == '>' if sys_is_le else sc == '<'
+    True
+    '''
+    code = code.lower()
+    if code is None:
+        return native_code
+    if code in aliases['little']:
+        return '<'
+    elif code in aliases['big']:
+        return '>'
+    elif code in aliases['native']:
+        return native_code
+    elif code in aliases['swapped']:
+        return swapped_code
+    else:
+        raise ValueError(
+            'We cannot handle byte order %s' % code)
+
+

Modified: branches/mb_mio_refactor/matlab/mio.py
===================================================================
--- branches/mb_mio_refactor/matlab/mio.py	2008-10-05 03:36:54 UTC (rev 4774)
+++ branches/mb_mio_refactor/matlab/mio.py	2008-10-05 04:49:23 UTC (rev 4775)
@@ -6,6 +6,7 @@
 
 import os
 import sys
+import warnings
 
 from miobase import get_matfile_version
 from mio4 import MatFile4Reader, MatFile4Writer
@@ -58,13 +59,13 @@
             raise IOError, 'Reader needs file name or open file-like object'
         byte_stream = file_name
 
-    mv = get_matfile_version(byte_stream)
-    if mv == '4':
+    mjv, mnv = get_matfile_version(byte_stream)
+    if mjv == 0:
         return MatFile4Reader(byte_stream, **kwargs)
-    elif mv == '5':
+    elif mjv == 1:
         return MatFile5Reader(byte_stream, **kwargs)
-    elif mv == '7':
-        raise NotImplementedError('Please use PyTables for matlab HDF files')
+    elif mjv == 2:
+        raise NotImplementedError('Please use PyTables for matlab v7.3 (HDF) files')
     else:
         raise TypeError('Did not recognize version %s' % mv)
     
@@ -94,10 +95,17 @@
     matlab_compatible   - returns matrices as would be loaded by matlab
                           (implies squeeze_me=False, chars_as_strings=False,
                           mat_dtype=True)
+    struct_as_record    - whether to load matlab structs as numpy record arrays, or
+    			  as old-style numpy arrays with dtype=object.
+                          (warns if not set, and defaults to False.  non-recarrays 
+                          cannot be exported via savemat.)
 
     v4 (Level 1.0), v6 and v7.1 matfiles are supported.
 
     '''
+    if not kwargs.get('struct_as_record', False):
+        warnings.warn("loading matlab structures as arrays of dtype=object is deprecated",
+                      DeprecationWarning, stacklevel=2)
     MR = mat_reader_factory(file_name, appendmat, **kwargs)
     matfile_dict = MR.get_variables()
     if mdict is not None:
@@ -131,7 +139,7 @@
     if format == '4':
         MW = MatFile4Writer(file_stream)
     elif format == '5':
-        MW = MatFile5Writer(file_stream)
+        MW = MatFile5Writer(file_stream, unicode_strings=True)
     else:
         raise ValueError, 'Format should be 4 or 5'
     MW.put_variables(mdict)

Modified: branches/mb_mio_refactor/matlab/mio4.py
===================================================================
--- branches/mb_mio_refactor/matlab/mio4.py	2008-10-05 03:36:54 UTC (rev 4774)
+++ branches/mb_mio_refactor/matlab/mio4.py	2008-10-05 04:49:23 UTC (rev 4775)
@@ -1,10 +1,14 @@
 ''' Classes for read / write of matlab (TM) 4 files
 '''
+import sys
 
-import numpy as N
+import numpy as np
 
-from miobase import *
+from miobase import MatFileReader, MatArrayReader, MatMatrixGetter, \
+     MatFileWriter, MatStreamWriter, spsparse
 
+SYS_LITTLE_ENDIAN = sys.byteorder == 'little'
+
 miDOUBLE = 0
 miSINGLE = 1
 miINT32 = 2
@@ -76,7 +80,7 @@
         header['mclass'] = T
         header['dims'] = (data['mrows'], data['ncols'])
         header['is_complex'] = data['imagf'] == 1
-        remaining_bytes = header['dtype'].itemsize * N.product(header['dims'])
+        remaining_bytes = header['dtype'].itemsize * np.product(header['dims'])
         if header['is_complex'] and not header['mclass'] == mxSPARSE_CLASS:
             remaining_bytes *= 2
         next_pos = self.mat_stream.tell() + remaining_bytes
@@ -109,10 +113,10 @@
         num_bytes = dt.itemsize
         for d in dims:
             num_bytes *= d
-        arr = N.ndarray(shape=dims,
-                      dtype=dt,
-                      buffer=self.mat_stream.read(num_bytes),
-                      order='F')
+        arr = np.ndarray(shape=dims,
+                         dtype=dt,
+                         buffer=self.mat_stream.read(num_bytes),
+                         order='F')
         if copy:
             arr = arr.copy()
         return arr
@@ -122,9 +126,9 @@
     def __init__(self, array_reader, header):
         super(Mat4FullGetter, self).__init__(array_reader, header)
         if header['is_complex']:
-            self.mat_dtype = N.dtype(N.complex128)
+            self.mat_dtype = np.dtype(np.complex128)
         else:
-            self.mat_dtype = N.dtype(N.float64)
+            self.mat_dtype = np.dtype(np.float64)
 
     def get_raw_array(self):
         if self.header['is_complex']:
@@ -137,12 +141,12 @@
 
 class Mat4CharGetter(Mat4MatrixGetter):
     def get_raw_array(self):
-        arr = self.read_array().astype(N.uint8)
+        arr = self.read_array().astype(np.uint8)
         # ascii to unicode
         S = arr.tostring().decode('ascii')
-        return N.ndarray(shape=self.header['dims'],
-                       dtype=N.dtype('U1'),
-                       buffer = N.array(S)).copy()
+        return np.ndarray(shape=self.header['dims'],
+                          dtype=np.dtype('U1'),
+                          buffer = np.array(S)).copy()
 
 
 class Mat4SparseGetter(Mat4MatrixGetter):
@@ -166,17 +170,17 @@
         res = self.read_array()
         tmp = res[:-1,:]
         dims = res[-1,0:2]
-        I = N.ascontiguousarray(tmp[:,0],dtype='intc') #fixes byte order also
-        J = N.ascontiguousarray(tmp[:,1],dtype='intc')
+        I = np.ascontiguousarray(tmp[:,0],dtype='intc') #fixes byte order also
+        J = np.ascontiguousarray(tmp[:,1],dtype='intc')
         I -= 1  # for 1-based indexing
         J -= 1
         if res.shape[1] == 3:
-            V = N.ascontiguousarray(tmp[:,2],dtype='float')
+            V = np.ascontiguousarray(tmp[:,2],dtype='float')
         else:
-            V = N.ascontiguousarray(tmp[:,2],dtype='complex')
+            V = np.ascontiguousarray(tmp[:,2],dtype='complex')
             V.imag = tmp[:,3]
-        if have_sparse:
-            return scipy.sparse.coo_matrix((V,(I,J)), dims)
+        if spsparse:
+            return spsparse.coo_matrix((V,(I,J)), dims)
         return (dims, I, J, V)
 
 
@@ -200,11 +204,11 @@
 
     def guess_byte_order(self):
         self.mat_stream.seek(0)
-        mopt = self.read_dtype(N.dtype('i4'))
+        mopt = self.read_dtype(np.dtype('i4'))
         self.mat_stream.seek(0)
         if mopt < 0 or mopt > 5000:
-            return ByteOrder.swapped_code
-        return ByteOrder.native_code
+            return SYS_LITTLE_ENDIAN and '>' or '<'
+        return SYS_LITTLE_ENDIAN and '<' or '>'
 
 
 class Mat4MatrixWriter(MatStreamWriter):
@@ -218,8 +222,8 @@
         '''
         if dims is None:
             dims = self.arr.shape
-        header = N.empty((), mdtypes_template['header'])
-        M = not ByteOrder.little_endian
+        header = np.empty((), mdtypes_template['header'])
+        M = not SYS_LITTLE_ENDIAN
         O = 0
         header['mopt'] = (M * 1000 +
                           O * 100 +
@@ -233,7 +237,7 @@
         self.write_string(self.name + '\0')
 
     def arr_to_2d(self):
-        self.arr = N.atleast_2d(self.arr)
+        self.arr = np.atleast_2d(self.arr)
         dims = self.arr.shape
         if len(dims) > 2:
             self.arr = self.arr.reshape(-1,dims[-1])
@@ -275,12 +279,12 @@
                           T=mxCHAR_CLASS)
         if self.arr.dtype.kind == 'U':
             # Recode unicode to ascii
-            n_chars = N.product(dims)
-            st_arr = N.ndarray(shape=(),
-                             dtype=self.arr_dtype_number(n_chars),
-                             buffer=self.arr)
+            n_chars = np.product(dims)
+            st_arr = np.ndarray(shape=(),
+                                dtype=self.arr_dtype_number(n_chars),
+                                buffer=self.arr)
             st = st_arr.item().encode('ascii')
-            self.arr = N.ndarray(shape=dims, dtype='S1', buffer=st)
+            self.arr = np.ndarray(shape=dims, dtype='S1', buffer=st)
         self.write_bytes(self.arr)
 
 
@@ -292,7 +296,7 @@
         '''
         A = self.arr.tocoo() #convert to sparse COO format (ijv)
         imagf = A.dtype.kind == 'c'
-        ijv = N.zeros((A.nnz + 1, 3+imagf), dtype='f8')
+        ijv = np.zeros((A.nnz + 1, 3+imagf), dtype='f8')
         ijv[:-1,0] = A.row
         ijv[:-1,1] = A.col
         ijv[:-1,0:2] += 1 # 1 based indexing
@@ -314,16 +318,16 @@
     arr         - array to write
     name        - name in matlab (TM) workspace
     '''
-    if have_sparse:
-        if scipy.sparse.issparse(arr):
+    if spsparse:
+        if spsparse.issparse(arr):
             return Mat4SparseWriter(stream, arr, name)
-    arr = N.array(arr)
+    arr = np.array(arr)
     dtt = arr.dtype.type
-    if dtt is N.object_:
+    if dtt is np.object_:
         raise TypeError, 'Cannot save object arrays in Mat4'
-    elif dtt is N.void:
+    elif dtt is np.void:
         raise TypeError, 'Cannot save void type arrays'
-    elif dtt in (N.unicode_, N.string_):
+    elif dtt in (np.unicode_, np.string_):
         return Mat4CharWriter(stream, arr, name)
     else:
         return Mat4NumericWriter(stream, arr, name)

Modified: branches/mb_mio_refactor/matlab/mio5.py
===================================================================
--- branches/mb_mio_refactor/matlab/mio5.py	2008-10-05 03:36:54 UTC (rev 4774)
+++ branches/mb_mio_refactor/matlab/mio5.py	2008-10-05 04:49:23 UTC (rev 4775)
@@ -26,17 +26,15 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
+import sys
 import zlib
+from cStringIO import StringIO
 from copy import copy as pycopy
-from cStringIO import StringIO
-import numpy as N
 
-from miobase import *
+import numpy as np
 
-try:  # Python 2.3 support
-    from sets import Set as set
-except:
-    pass
+from miobase import MatFileReader, MatArrayReader, MatMatrixGetter, \
+     MatFileWriter, MatStreamWriter, spsparse
 
 miINT8 = 1
 miUINT8 = 2
@@ -56,6 +54,10 @@
 
 mxCELL_CLASS = 1
 mxSTRUCT_CLASS = 2
+# The March 2008 edition of "Matlab 7 MAT-File Format" says that
+# mxOBJECT_CLASS = 3, whereas matrix.h says that mxLOGICAL = 3.
+# Matlab 2008a appears to save logicals as type 9, so we assume that
+# the document is correct.  See type 18, below.
 mxOBJECT_CLASS = 3
 mxCHAR_CLASS = 4
 mxSPARSE_CLASS = 5
@@ -67,6 +69,14 @@
 mxUINT16_CLASS = 11
 mxINT32_CLASS = 12
 mxUINT32_CLASS = 13
+# The following are not in the March 2008 edition of "Matlab 7
+# MAT-File Format," but were guessed from matrix.h.
+mxINT64_CLASS = 14
+mxUINT64_CLASS = 15
+mxFUNCTION_CLASS = 16
+# Not doing anything with these at the moment.
+mxOPAQUE_CLASS = 17
+mxOBJECT_CLASS_FROM_MATRIX_H = 18
 
 mdtypes_template = {
     miINT8: 'i1',
@@ -87,6 +97,7 @@
                     ('version', 'u2'),
                     ('endian_test', 'S2')],
     'tag_full': [('mdtype', 'u4'), ('byte_count', 'u4')],
+    'tag_smalldata':[('byte_count_mdtype', 'u4'), ('data', 'S4')],
     'array_flags': [('data_type', 'u4'),
                     ('byte_count', 'u4'),
                     ('flags_class','u4'),
@@ -101,6 +112,8 @@
     mxUINT16_CLASS: 'u2',
     mxINT32_CLASS: 'i4',
     mxUINT32_CLASS: 'u4',
+    mxINT64_CLASS: 'i8',
+    mxUINT64_CLASS: 'u8',
     mxSINGLE_CLASS: 'f4',
     mxDOUBLE_CLASS: 'f8',
     }
@@ -116,9 +129,11 @@
     'i1': miINT8,
     'i2': miINT16,
     'i4': miINT32,
+    'i8': miINT64,
     'u1': miUINT8,
+    'u2': miUINT16,
     'u4': miUINT32,
-    'u2': miUINT16,
+    'u8': miUINT64,
     'S1': miUINT8,
     'U1': miUTF16,
     }
@@ -131,8 +146,10 @@
     'c16': mxDOUBLE_CLASS,
     'f4': mxSINGLE_CLASS,
     'c8': mxSINGLE_CLASS,
+    'i8': mxINT64_CLASS,
     'i4': mxINT32_CLASS,
     'i2': mxINT16_CLASS,
+    'u8': mxUINT64_CLASS,
     'u2': mxUINT16_CLASS,
     'u1': mxUINT8_CLASS,
     'S1': mxUINT8_CLASS,
@@ -162,16 +179,10 @@
     mxUINT16_CLASS,
     mxINT32_CLASS,
     mxUINT32_CLASS,
+    mxINT64_CLASS,
+    mxUINT64_CLASS,
     )
 
-class mat_struct(object):
-    ''' Placeholder for holding read data from structs '''
-    pass
-
-class mat_obj(object):
-    ''' Placeholder for holding read data from objects '''
-    pass
-
 class Mat5ArrayReader(MatArrayReader):
     ''' Class to get Mat5 arrays
 
@@ -179,36 +190,55 @@
     factory function
     '''
 
-    def __init__(self, mat_stream, dtypes, processor_func, codecs, class_dtypes):
+    def __init__(self, mat_stream, dtypes, processor_func, codecs, class_dtypes, struct_as_record):
         super(Mat5ArrayReader, self).__init__(mat_stream,
                                               dtypes,
-                                              processor_func,
-                                              )
+                                              processor_func)
         self.codecs = codecs
         self.class_dtypes = class_dtypes
+        self.struct_as_record = struct_as_record
 
     def read_element(self, copy=True):
-        mdtype, byte_count, buf = tagparse(self.mat_stream, swapf)
-        if mdtype == miMATRIX:
-            # Can this use buf or not?
-            return self.current_getter(byte_count).get_array()
-        elif mdtype in self.codecs: # encoded char data
+        raw_tag = self.mat_stream.read(8)
+        tag = np.ndarray(shape=(),
+                         dtype=self.dtypes['tag_full'],
+                         buffer=raw_tag)
+        mdtype = tag['mdtype'].item()
+        # Byte count if this is small data element
+        byte_count = mdtype >> 16
+        if byte_count: # small data element format
+            if byte_count > 4:
+                raise ValueError, 'Too many bytes for sde format'
+            mdtype = mdtype & 0xFFFF
+            if mdtype == miMATRIX:
+                raise TypeError('Cannot have matrix in SDE format')
+            raw_str = raw_tag[4:byte_count+4]
+        else: # regular element
+            byte_count = tag['byte_count'].item()
+            # Deal with miMATRIX type (cannot pass byte string)
+            if mdtype == miMATRIX:
+                return self.current_getter(byte_count).get_array()
+            # All other types can be read from string
+            raw_str = self.mat_stream.read(byte_count)
+            # Seek to next 64-bit boundary
+            mod8 = byte_count % 8
+            if mod8:
+                self.mat_stream.seek(8 - mod8, 1)
+            
+        if mdtype in self.codecs: # encoded char data
             codec = self.codecs[mdtype]
             if not codec:
                 raise TypeError, 'Do not support encoding %d' % mdtype
-            el = buf.decode(codec)
+            el = raw_str.decode(codec)
         else: # numeric data
             dt = self.dtypes[mdtype]
             el_count = byte_count // dt.itemsize
-            el = N.ndarray(shape=(el_count,),
-                         dtype=dt,
-                         buffer=buf)
+            el = np.ndarray(shape=(el_count,),
+                            dtype=dt,
+                            buffer=raw_str)
             if copy:
                 el = el.copy()
-        # Seek to next 64-bit boundary
-        mod8 = byte_count % 8
-        if mod8:
-            self.mat_stream.seek(8 - mod8, 1)
+       
         return el
 
     def matrix_getter_factory(self):
@@ -254,9 +284,11 @@
         if mc == mxCELL_CLASS:
             return Mat5CellMatrixGetter(self, header)
         if mc == mxSTRUCT_CLASS:
-            return Mat5StructMatrixGetter(self, header)
+            return Mat5StructMatrixGetter(self, header, self.struct_as_record)
         if mc == mxOBJECT_CLASS:
             return Mat5ObjectMatrixGetter(self, header)
+        if mc == mxFUNCTION_CLASS:
+            return Mat5FunctionMatrixGetter(self, header)
         raise TypeError, 'No reader for class code %s' % mc
 
 
@@ -274,7 +306,8 @@
             array_reader.dtypes,
             array_reader.processor_func,
             array_reader.codecs,
-            array_reader.class_dtypes)
+            array_reader.class_dtypes,
+            array_reader.struct_as_record)
 
 
 class Mat5MatrixGetter(MatMatrixGetter):
@@ -302,11 +335,12 @@
         self.mat_stream = array_reader.mat_stream
         self.data_position = self.mat_stream.tell()
         self.header = {}
+        self.name = ''
         self.is_global = False
         self.mat_dtype = 'f8'
 
     def get_raw_array(self):
-        return N.array([[]])
+        return np.array([[]])
 
 
 class Mat5NumericMatrixGetter(Mat5MatrixGetter):
@@ -314,7 +348,7 @@
     def __init__(self, array_reader, header):
         super(Mat5NumericMatrixGetter, self).__init__(array_reader, header)
         if header['is_logical']:
-            self.mat_dtype = N.dtype('bool')
+            self.mat_dtype = np.dtype('bool')
         else:
             self.mat_dtype = self.class_dtypes[header['mclass']]
 
@@ -326,10 +360,10 @@
             res = res + (res_j * 1j)
         else:
             res = self.read_element()
-        return N.ndarray(shape=self.header['dims'],
-                       dtype=res.dtype,
-                       buffer=res,
-                       order='F')
+        return np.ndarray(shape=self.header['dims'],
+                          dtype=res.dtype,
+                          buffer=res,
+                          order='F')
 
 
 class Mat5SparseMatrixGetter(Mat5MatrixGetter):
@@ -360,39 +394,38 @@
         nnz = indptr[-1]
         rowind = rowind[:nnz]
         data   = data[:nnz]
-        if have_sparse:
-            from scipy.sparse import csc_matrix
-            return csc_matrix((data,rowind,indptr), shape=(M,N))
+        if spsparse:
+            return spsparse.csc_matrix((data,rowind,indptr), shape=(M,N))
         else:
-            return (dims, data, rowind, indptr)
+            return ((M,N), data, rowind, indptr)
 
 
 class Mat5CharMatrixGetter(Mat5MatrixGetter):
     def get_raw_array(self):
         res = self.read_element()
         # Convert non-string types to unicode
-        if isinstance(res, N.ndarray):
-            if res.dtype.type == N.uint16:
+        if isinstance(res, np.ndarray):
+            if res.dtype.type == np.uint16:
                 codec = miUINT16_codec
                 if self.codecs['uint16_len'] == 1:
-                    res = res.astype(N.uint8)
-            elif res.dtype.type in (N.uint8, N.int8):
+                    res = res.astype(np.uint8)
+            elif res.dtype.type in (np.uint8, np.int8):
                 codec = 'ascii'
             else:
                 raise TypeError, 'Did not expect type %s' % res.dtype
             res = res.tostring().decode(codec)
-        return N.ndarray(shape=self.header['dims'],
-                       dtype=N.dtype('U1'),
-                       buffer=N.array(res),
-                       order='F').copy()
+        return np.ndarray(shape=self.header['dims'],
+                          dtype=np.dtype('U1'),
+                          buffer=np.array(res),
+                          order='F').copy()
 
 
 class Mat5CellMatrixGetter(Mat5MatrixGetter):
     def get_raw_array(self):
         # Account for fortran indexing of cells
         tupdims = tuple(self.header['dims'][::-1])
-        length = N.product(tupdims)
-        result = N.empty(length, dtype=object)
+        length = np.product(tupdims)
+        result = np.empty(length, dtype=object)
         for i in range(length):
             result[i] = self.get_item()
         return result.reshape(tupdims).T
@@ -400,47 +433,97 @@
     def get_item(self):
         return self.read_element()
 
+class mat_struct(object):
+    ''' Placeholder for holding read data from structs '''
+    pass
 
-class Mat5StructMatrixGetter(Mat5CellMatrixGetter):
-    def __init__(self, *args, **kwargs):
-        super(Mat5StructMatrixGetter, self).__init__(*args, **kwargs)
-        self.obj_template = mat_struct()
+class Mat5StructMatrixGetter(Mat5MatrixGetter):
+    def __init__(self, array_reader, header, struct_as_record):
+        super(Mat5StructMatrixGetter, self).__init__(array_reader, header)
+        self.struct_as_record = struct_as_record
 
     def get_raw_array(self):
         namelength = self.read_element()[0]
-        # get field names
         names = self.read_element()
-        splitnames = [names[i:i+namelength] for i in \
-                      xrange(0,len(names),namelength)]
-        self.obj_template._fieldnames = [x.tostring().strip('\x00')
-                                        for x in splitnames]
-        return super(Mat5StructMatrixGetter, self).get_raw_array()
+        field_names = [names[i:i+namelength].tostring().strip('\x00')
+                       for i in xrange(0,len(names),namelength)]
+        tupdims = tuple(self.header['dims'][::-1])
+        length = np.product(tupdims)
+        result = np.empty(length, dtype=[(field_name, object) 
+                                         for field_name in field_names])
+        for i in range(length):
+            for field_name in field_names:
+                result[i][field_name] = self.read_element()
+        
+        if not self.struct_as_record:
+            # Backward compatibility with previous format
+            self.obj_template = mat_struct()
+            self.obj_template._fieldnames = field_names
+            newresult = np.empty(length, dtype=object)
+            for i in range(length):
+                item = pycopy(self.obj_template)
+                for name in field_names:
+                    item.__dict__[name] = result[i][name]
+                newresult[i] = item
+            result = newresult
+        
+        return result.reshape(tupdims).T
 
-    def get_item(self):
-        item = pycopy(self.obj_template)
-        for element in item._fieldnames:
-            item.__dict__[element]  = self.read_element()
-        return item
+class MatlabObject(object):
+    ''' Class to contain read data from matlab objects '''
+    def __init__(self, classname, field_names):
+        self.__dict__['classname'] = classname
+        self.__dict__['mobj_recarray'] = np.empty((1,1), dtype=[(field_name, object) 
+                                            for field_name in field_names])
 
+    def __getattr__(self, name):
+        mobj_recarray = self.__dict__['mobj_recarray']
+        if name in mobj_recarray.dtype.fields:
+            return mobj_recarray[0,0][name]
+        else:
+            raise AttributeError, "no field named %s in MatlabObject"%(name)
 
-class Mat5ObjectMatrixGetter(Mat5StructMatrixGetter):
-    def __init__(self, *args, **kwargs):
-        super(Mat5StructMatrixGetter, self).__init__(*args, **kwargs)
-        self.obj_template = mat_obj()
+    def __setattr__(self, name, value):
+        if name in self.__dict__['mobj_recarray'].dtype.fields:
+            self.__dict__['mobj_recarray'][0,0][name] = value
+        else:
+            self.__dict__[name] = value
+        
 
-    def get_raw_array(self):
-        self.obj_template._classname = self.read_element().tostring()
-        return super(Mat5ObjectMatrixGetter, self).get_raw_array()
+class Mat5ObjectMatrixGetter(Mat5MatrixGetter):
+    def get_array(self):
+        '''Matlab ojects are essentially structs, with an extra field, the classname.'''
+        classname = self.read_element().tostring()
+        namelength = self.read_element()[0]
+        names = self.read_element()
+        field_names = [names[i:i+namelength].tostring().strip('\x00')
+                       for i in xrange(0,len(names),namelength)]
+        result = MatlabObject(classname, field_names)
 
+        for field_name in field_names:
+            result.__setattr__(field_name, self.read_element())
 
+        return result
+
+
+class MatlabFunctionMatrix:
+    ''' Opaque object representing an array of function handles. '''
+    def __init__(self, arr):
+        self.arr = arr
+
+class Mat5FunctionMatrixGetter(Mat5CellMatrixGetter):
+    def get_array(self):
+        return MatlabFunctionMatrix(self.get_raw_array())
+
+
 class MatFile5Reader(MatFileReader):
     ''' Reader for Mat 5 mat files
-
     Adds the following attribute to base class
-
+    
     uint16_codec       - char codec to use for uint16 char arrays
                           (defaults to system default codec)
    '''
+
     def __init__(self,
                  mat_stream,
                  byte_order=None,
@@ -448,6 +531,7 @@
                  squeeze_me=False,
                  chars_as_strings=True,
                  matlab_compatible=False,
+                 struct_as_record=False,
                  uint16_codec=None
                  ):
         self.codecs = {}
@@ -457,6 +541,7 @@
             None,
             None,
             None,
+            struct_as_record
             )
         super(MatFile5Reader, self).__init__(
             mat_stream,
@@ -533,7 +618,7 @@
 
 class Mat5MatrixWriter(MatStreamWriter):
 
-    mat_tag = N.zeros((), mdtypes_template['tag_full'])
+    mat_tag = np.zeros((), mdtypes_template['tag_full'])
     mat_tag['mdtype'] = miMATRIX
 
     def __init__(self, file_stream, arr, name, is_global=False):
@@ -545,20 +630,32 @@
 
     def write_element(self, arr, mdtype=None):
         # write tag, data
-        tag = N.zeros((), mdtypes_template['tag_full'])
         if mdtype is None:
-            tag['mdtype'] = np_to_mtypes[arr.dtype.str[1:]]
+            mdtype = np_to_mtypes[arr.dtype.str[1:]]
+        byte_count = arr.size*arr.itemsize
+        if byte_count <= 4:
+            self.write_smalldata_element(arr, mdtype, byte_count)
         else:
-            tag['mdtype'] = mdtype
+            self.write_regular_element(arr, mdtype, byte_count)
 
-        tag['byte_count'] = arr.size*arr.itemsize
+    def write_smalldata_element(self, arr, mdtype, byte_count):
+        # write tag with embedded data
+        tag = np.zeros((), mdtypes_template['tag_smalldata'])
+        tag['byte_count_mdtype'] = (byte_count << 16) + mdtype
+        # if arr.tostring is < 4, the element will be zero-padded as needed.
+        tag['data'] = arr.tostring(order='F')
+        self.write_dtype(tag)
+
+    def write_regular_element(self, arr, mdtype, byte_count):
+        # write tag, data
+        tag = np.zeros((), mdtypes_template['tag_full'])
+        tag['mdtype'] = mdtype
+        tag['byte_count'] = byte_count
         padding = (8 - tag['byte_count']) % 8
-
         self.write_dtype(tag)
         self.write_bytes(arr)
-
         # pad to next 64-bit boundary
-        self.write_bytes(N.zeros((padding,),'u1'))
+        self.write_bytes(np.zeros((padding,),'u1'))
 
     def write_header(self, mclass,
                      is_global=False,
@@ -568,14 +665,14 @@
         ''' Write header for given data options
         mclass      - mat5 matrix class
         is_global   - True if matrix is global
-        is_complex  - True is matrix is complex
+        is_complex  - True if matrix is complex
         is_logical  - True if matrix is logical
         nzmax        - max non zero elements for sparse arrays
         '''
         self._mat_tag_pos = self.file_stream.tell()
         self.write_dtype(self.mat_tag)
         # write array flags (complex, global, logical, class, nzmax)
-        af = N.zeros((), mdtypes_template['array_flags'])
+        af = np.zeros((), mdtypes_template['array_flags'])
         af['data_type'] = miUINT32
         af['byte_count'] = 8
         flags = is_complex << 3 | is_global << 2 | is_logical << 1
@@ -584,13 +681,13 @@
         self.write_dtype(af)
         # write array shape
         if self.arr.ndim < 2:
-            new_arr = N.atleast_2d(self.arr)
+            new_arr = np.atleast_2d(self.arr)
             if type(new_arr) != type(self.arr):
                 raise ValueError("Array should be 2-dimensional.")
             self.arr = new_arr
-        self.write_element(N.array(self.arr.shape, dtype='i4'))
+        self.write_element(np.array(self.arr.shape, dtype='i4'))
         # write name
-        self.write_element(N.array([ord(c) for c in self.name], 'i1'))
+        self.write_element(np.array([ord(c) for c in self.name], 'i1'))
 
     def update_matrix_tag(self):
         curr_pos = self.file_stream.tell()
@@ -604,7 +701,6 @@
 
 
 class Mat5NumericWriter(Mat5MatrixWriter):
-
     def write(self):
         imagf = self.arr.dtype.kind == 'c'
         try:
@@ -630,12 +726,12 @@
         self.write_header(mclass=mxCHAR_CLASS)
         if self.arr.dtype.kind == 'U':
             # Recode unicode using self.codec
-            n_chars = N.product(self.arr.shape)
-            st_arr = N.ndarray(shape=(),
-                             dtype=self.arr_dtype_number(n_chars),
-                             buffer=self.arr)
+            n_chars = np.product(self.arr.shape)
+            st_arr = np.ndarray(shape=(),
+                                dtype=self.arr_dtype_number(n_chars),
+                                buffer=self.arr)
             st = st_arr.item().encode(self.codec)
-            self.arr = N.ndarray(shape=(len(st)), dtype='u1', buffer=st)
+            self.arr = np.ndarray(shape=(len(st)), dtype='u1', buffer=st)
         self.write_element(self.arr,mdtype=miUTF8)
         self.update_matrix_tag()
 
@@ -644,10 +740,8 @@
 
 
 class Mat5SparseWriter(Mat5MatrixWriter):
-
     def write(self):
         ''' Sparse matrices are 2D
-
         '''
         A = self.arr.tocsc() # convert to sparse CSC format
         A.sort_indices()     # MATLAB expects sorted row indices
@@ -664,6 +758,80 @@
         self.update_matrix_tag()
 
 
+class Mat5CompositeWriter(Mat5MatrixWriter):
+    def __init__(self, file_stream, arr, name, is_global=False, unicode_strings=False):
+        super(Mat5CompositeWriter, self).__init__(file_stream, arr, name, is_global)
+        self.unicode_strings = unicode_strings
+        
+
+class Mat5CellWriter(Mat5CompositeWriter):
+    def write(self):
+        self.write_header(mclass=mxCELL_CLASS)
+        # loop over data, column major
+        A = np.atleast_2d(self.arr).flatten('F')
+        MWG = Mat5WriterGetter(self.file_stream, self.unicode_strings)
+        for el in A:
+            MW = MWG.matrix_writer_factory(el, '')
+            MW.write()
+        self.update_matrix_tag()
+
+class Mat5FunctionWriter(Mat5CompositeWriter):
+    def __init__(self, file_stream, arr, name, is_global=False, unicode_strings=False):
+        super(Mat5FunctionWriter, self).__init__(file_stream, arr.arr, name, is_global)
+
+    def write(self):
+        self.write_header(mclass=mxFUNCTION_CLASS)
+        # loop over data, column major
+        A = np.atleast_2d(self.arr).flatten('F')
+        MWG = Mat5WriterGetter(self.file_stream, self.unicode_strings)
+        for el in A:
+            MW = MWG.matrix_writer_factory(el, '')
+            MW.write()
+        self.update_matrix_tag()
+
+
+class Mat5StructWriter(Mat5CompositeWriter):
+    def write(self):
+        self.write_header(mclass=mxSTRUCT_CLASS)
+        
+        # write fieldnames
+        fieldnames = [f[0] for f in self.arr.dtype.descr]
+        self.write_element(np.array([32], dtype='i4'))
+        self.write_element(np.array(fieldnames, dtype='S32'), mdtype=miINT8)
+        
+        A = np.atleast_2d(self.arr).flatten('F')
+        MWG = Mat5WriterGetter(self.file_stream, self.unicode_strings)
+        for el in A:
+            for f in fieldnames:
+                MW = MWG.matrix_writer_factory(el[f], '')
+                MW.write()
+        self.update_matrix_tag()
+
+class Mat5ObjectWriter(Mat5CompositeWriter):
+    def __init__(self, file_stream, arr, name, is_global=False, unicode_strings=False):
+        super(Mat5ObjectWriter, self).__init__(file_stream, arr.__dict__['mobj_recarray'], name, is_global)
+        self.classname = arr.classname
+
+    def write(self):
+        self.write_header(mclass=mxOBJECT_CLASS)
+
+        # write classnames
+        self.write_element(np.array(self.classname, dtype='S'), mdtype=miINT8)
+
+        # write fieldnames
+        fieldnames = [f[0] for f in self.arr.dtype.descr]
+        self.write_element(np.array([32], dtype='i4'))
+        self.write_element(np.array(fieldnames, dtype='S32'), mdtype=miINT8)
+        
+        A = np.atleast_2d(self.arr).flatten('F')
+        MWG = Mat5WriterGetter(self.file_stream, self.unicode_strings)
+        for el in A:
+            for f in fieldnames:
+                MW = MWG.matrix_writer_factory(el[f], '')
+                MW.write()
+        self.update_matrix_tag()
+
+    
 class Mat5WriterGetter(object):
     ''' Wraps stream and options, provides methods for getting Writer objects '''
     def __init__(self, stream, unicode_strings):
@@ -679,18 +847,21 @@
         arr         - array to write
         name        - name in matlab (TM) workspace
         '''
-        if have_sparse:
-            if scipy.sparse.issparse(arr):
+        if spsparse:
+            if spsparse.issparse(arr):
                 return Mat5SparseWriter(self.stream, arr, name, is_global)
-        arr = N.array(arr)
+            
+        if isinstance(arr, MatlabFunctionMatrix):
+            return Mat5FunctionWriter(self.stream, arr, name, is_global, self.unicode_strings)
+        if isinstance(arr, MatlabObject):
+            return Mat5ObjectWriter(self.stream, arr, name, is_global, self.unicode_strings)
+        
+        arr = np.array(arr)
         if arr.dtype.hasobject:
-            types, arr_type = self.classify_mobjects(arr)
-            if arr_type == 'c':
-                return Mat5CellWriter(self.stream, arr, name, is_global, types)
-            elif arr_type == 's':
-                return Mat5StructWriter(self.stream, arr, name, is_global)
-            elif arr_type == 'o':
-                return Mat5ObjectWriter(self.stream, arr, name, is_global)
+            if arr.dtype.fields == None:
+                return Mat5CellWriter(self.stream, arr, name, is_global, self.unicode_strings)
+            else:
+                return Mat5StructWriter(self.stream, arr, name, is_global, self.unicode_strings)
         if arr.dtype.kind in ('U', 'S'):
             if self.unicode_strings:
                 return Mat5UniCharWriter(self.stream, arr, name, is_global)
@@ -699,47 +870,6 @@
         else:
             return Mat5NumericWriter(self.stream, arr, name, is_global)
 
-    def classify_mobjects(self, objarr):
-        ''' Function to classify objects passed for writing
-        returns
-        types         - S1 array of same shape as objarr with codes for each object
-                        i  - invalid object
-                        a  - ndarray
-                        s  - matlab struct
-                        o  - matlab object
-        arr_type       - one of
-                        c  - cell array
-                        s  - struct array
-                        o  - object array
-        '''
-        n = objarr.size
-        types = N.empty((n,), dtype='S1')
-        types[:] = 'i'
-        type_set = set()
-        flato = objarr.flat
-        for i in range(n):
-            obj = flato[i]
-            if isinstance(obj, N.ndarray):
-                types[i] = 'a'
-                continue
-            try:
-                fns = tuple(obj._fieldnames)
-            except AttributeError:
-                continue
-            try:
-                cn = obj._classname
-            except AttributeError:
-                types[i] = 's'
-                type_set.add(fns)
-                continue
-            types[i] = 'o'
-            type_set.add((cn, fns))
-        arr_type = 'c'
-        if len(set(types))==1 and len(type_set) == 1:
-            arr_type = types[0]
-        return types.reshape(objarr.shape), arr_type
-
-
 class MatFile5Writer(MatFileWriter):
     ''' Class for writing mat5 files '''
     def __init__(self, file_stream,
@@ -757,11 +887,11 @@
             unicode_strings)
         # write header
         import os, time
-        hdr =  N.zeros((), mdtypes_template['file_header'])
+        hdr =  np.zeros((), mdtypes_template['file_header'])
         hdr['description']='MATLAB 5.0 MAT-file Platform: %s, Created on: %s' % (
                             os.name,time.asctime())
         hdr['version']= 0x0100
-        hdr['endian_test']=N.ndarray(shape=(),dtype='S2',buffer=N.uint16(0x4d49))
+        hdr['endian_test']=np.ndarray(shape=(),dtype='S2',buffer=np.uint16(0x4d49))
         file_stream.write(hdr.tostring())
 
     def get_unicode_strings(self):
@@ -775,6 +905,8 @@
 
     def put_variables(self, mdict):
         for name, var in mdict.items():
+            if name[0] == '_':
+                continue
             is_global = name in self.global_vars
             self.writer_getter.rewind()
             self.writer_getter.matrix_writer_factory(
@@ -785,7 +917,7 @@
             stream = self.writer_getter.stream
             if self.do_compression:
                 str = zlib.compress(stream.getvalue(stream.tell()))
-                tag = N.empty((), mdtypes_template['tag_full'])
+                tag = np.empty((), mdtypes_template['tag_full'])
                 tag['mdtype'] = miCOMPRESSED
                 tag['byte_count'] = len(str)
                 self.file_stream.write(tag.tostring() + str)

Modified: branches/mb_mio_refactor/matlab/miobase.py
===================================================================
--- branches/mb_mio_refactor/matlab/miobase.py	2008-10-05 03:36:54 UTC (rev 4774)
+++ branches/mb_mio_refactor/matlab/miobase.py	2008-10-05 04:49:23 UTC (rev 4775)
@@ -6,15 +6,16 @@
 
 import sys
 
-import numpy as N
+import numpy as np
 
+import byteordercodes as sibc
+
+# sparse module if available
 try:
-    import scipy.sparse
-    have_sparse = 1
+    import scipy.sparse as spsparse
 except ImportError:
-    have_sparse = 0
+    spsparse = None
 
-
 def small_product(arr):
     ''' Faster than product for small arrays '''
     res = 1
@@ -23,57 +24,67 @@
     return res
 
 def get_matfile_version(fileobj):
-    ''' Return '4', '5', or '7' depending on apparent mat file type
-    Inputs
-    fileobj       - file object implementing seek() and read()
-    Outputs
-    version_str   - one of (strings) 4, 5, or 7
-    
+    ''' Return major, minor tuple depending on apparent mat file type
+
+    Where:
+
+     #. 0,x -> version 4 format mat files
+     #. 1,x -> version 5 format mat files
+     #. 2,x -> version 7.3 format mat files (HDF format)
+     
+    Parameters
+    ----------
+    fileobj : {file-like}
+              object implementing seek() and read()
+
+    Returns
+    -------
+    major_version : {0, 1, 2}
+                    major matlab file format version
+    minor_version : int
+                    major matlab file format version
+
+    Notes
+    -----
     Has the side effect of setting the file read pointer to 0
     '''
     # Mat4 files have a zero somewhere in first 4 bytes
     fileobj.seek(0)
-    mopt_bytes = N.ndarray(shape=(4,),
-                           dtype=N.uint8,
+    mopt_bytes = np.ndarray(shape=(4,),
+                           dtype=np.uint8,
                            buffer = fileobj.read(4))
     if 0 in mopt_bytes:
         fileobj.seek(0)
-        return '4'
-    # For 5 or 7 we need to read an integer in the header
-    # bytes 124 through 128 contain a version integer
-    # and an endian test string
+        return (0,0)
+    
+    # For 5 format or 7.3 format we need to read an integer in the
+    # header. Bytes 124 through 128 contain a version integer and an
+    # endian test string
     fileobj.seek(124)
     tst_str = fileobj.read(4)
     fileobj.seek(0)
     maj_ind = int(tst_str[2] == 'I')
-    verb = ord(tst_str[maj_ind])
-    if verb == 1:
-        return '5'
-    elif verb == 2:
-        return '7'
-    raise ValueError('Unknown mat file type, version %d' % verb)
+    maj_val = ord(tst_str[maj_ind])
+    min_val = ord(tst_str[1-maj_ind])
+    ret = (maj_val, min_val)
+    if maj_val in (1, 2):
+        return ret
+    else:
+        raise ValueError('Unknown mat file type, version %s' % ret)
 
 
 class ByteOrder(object):
     ''' Namespace for byte ordering '''
-    little_endian = sys.byteorder == 'little'
-    native_code = little_endian and '<' or '>'
-    swapped_code = little_endian and '>' or '<'
+    little_endian = sibc.sys_is_le
+    native_code = sibc.native_code
+    swapped_code = sibc.swapped_code
+    to_numpy_code = sibc.to_numpy_code
 
-    def to_numpy_code(code):
-        if code is None:
-            return ByteOrder.native_code
-        if code in ('little', '<', 'l', 'L'):
-            return '<'
-        elif code in ('BIG', '>', 'B', 'b'):
-            return '>'
-        elif code in ('native', '='):
-            return ByteOrder.native_code
-        elif code in ('swapped'):
-            return ByteOrder.swapped_code
-        else:
-            raise ValueError, 'We cannot handle byte order %s' % byte_order
-    to_numpy_code = staticmethod(to_numpy_code)
+ByteOrder = np.deprecate_with_doc("""
+We no longer use the ByteOrder class, and deprecate it; we will remove
+it in future versions of scipy.  Please use the
+scipy.io.byteordercodes module instead.
+""")(ByteOrder)
 
 
 class MatStreamAgent(object):
@@ -103,10 +114,10 @@
         a_dtype is assumed to be correct endianness
         '''
         num_bytes = a_dtype.itemsize
-        arr = N.ndarray(shape=(),
-                        dtype=a_dtype,
-                        buffer=self.mat_stream.read(num_bytes),
-                        order='F')
+        arr = np.ndarray(shape=(),
+                         dtype=a_dtype,
+                         buffer=self.mat_stream.read(num_bytes),
+                         order='F')
         return arr
 
     def read_ztstring(self, num_bytes):
@@ -128,6 +139,7 @@
     matlab_compatible  - returns matrices as would be loaded by matlab
                          (implies squeeze_me=False, chars_as_strings=False
                          mat_dtype=True)
+    struct_as_record   - return strutures as numpy records (only from v5 files)
 
     To make this class functional, you will need to override the
     following methods:
@@ -136,12 +148,14 @@
     matrix_getter_factory   - gives object to fetch next matrix from stream
     guess_byte_order        - guesses file byte order from file
     """
+
     def __init__(self, mat_stream,
                  byte_order=None,
                  mat_dtype=False,
                  squeeze_me=False,
                  chars_as_strings=True,
                  matlab_compatible=False,
+                 struct_as_record=False
                  ):
         # Initialize stream
         self.mat_stream = mat_stream
@@ -197,7 +211,7 @@
     def get_order_code(self):
         return self._order_code
     def set_order_code(self, order_code):
-        order_code = ByteOrder.to_numpy_code(order_code)
+        order_code = sibc.to_numpy_code(order_code)
         self._order_code = order_code
         self.set_dtypes()
     order_code = property(get_order_code,
@@ -212,8 +226,7 @@
     def convert_dtypes(self, dtype_template):
         dtypes = dtype_template.copy()
         for k in dtypes:
-            dtypes[k] = N.dtype(dtypes[k]).newbyteorder(
-                self.order_code)
+            dtypes[k] = np.dtype(dtypes[k]).newbyteorder(self.order_code)
         return dtypes
 
     def matrix_getter_factory(self):
@@ -255,7 +268,7 @@
                     str_arr = arr.reshape(
                         (small_product(n_dims),
                          dims[-1]))
-                    arr = N.empty(n_dims, dtype=object)
+                    arr = np.empty(n_dims, dtype='U%d' % dims[-1])
                     for i in range(0, n_dims[-1]):
                         arr[...,i] = self.chars_to_str(str_arr[i])
                 else: # return string
@@ -266,9 +279,9 @@
                 if getter.mat_dtype is not None:
                     arr = arr.astype(getter.mat_dtype)
             if self.squeeze_me:
-                arr = N.squeeze(arr)
+                arr = np.squeeze(arr)
                 if not arr.size:
-                    arr = N.array([])
+                    arr = np.array([])
                 elif not arr.shape: # 0d coverted to scalar
                     arr = arr.item()
             return arr
@@ -276,10 +289,10 @@
 
     def chars_to_str(self, str_arr):
         ''' Convert string array to string '''
-        dt = N.dtype('U' + str(small_product(str_arr.shape)))
-        return N.ndarray(shape=(),
-                       dtype = dt,
-                       buffer = str_arr.copy()).item()
+        dt = np.dtype('U' + str(small_product(str_arr.shape)))
+        return np.ndarray(shape=(),
+                          dtype = dt,
+                          buffer = str_arr.copy()).item()
 
     def get_variables(self, variable_names=None):
         ''' get variables from stream as dictionary
@@ -380,7 +393,7 @@
 
     def arr_dtype_number(self, num):
         ''' Return dtype for given number of items per element'''
-        return N.dtype(self.arr.dtype.str[:2] + str(num))
+        return np.dtype(self.arr.dtype.str[:2] + str(num))
 
     def arr_to_chars(self):
         ''' Convert string array to char array '''
@@ -388,9 +401,9 @@
         if not dims:
             dims = [1]
         dims.append(int(self.arr.dtype.str[2:]))
-        self.arr = N.ndarray(shape=dims,
-                           dtype=self.arr_dtype_number(1),
-                           buffer=self.arr)
+        self.arr = np.ndarray(shape=dims,
+                              dtype=self.arr_dtype_number(1),
+                              buffer=self.arr)
 
     def write_bytes(self, arr):
         self.file_stream.write(arr.tostring(order='F'))

Added: trunk/scipy/io/matlab/tests/data/testhdf5_7.4_GLNX86.mat
===================================================================
(Binary files differ)


Property changes on: trunk/scipy/io/matlab/tests/data/testhdf5_7.4_GLNX86.mat
___________________________________________________________________
Name: svn:mime-type
   + application/octet-stream

Modified: trunk/scipy/io/matlab/tests/save_test.m
===================================================================
--- trunk/scipy/io/matlab/tests/save_test.m	2008-10-05 03:36:54 UTC (rev 4774)
+++ trunk/scipy/io/matlab/tests/save_test.m	2008-10-05 04:49:23 UTC (rev 4775)
@@ -3,4 +3,4 @@
   
 global FILEPREFIX FILESUFFIX
 eval([test_name ' = v;']);
-save([FILEPREFIX test_name FILESUFFIX], test_name)
\ No newline at end of file
+save([FILEPREFIX test_name FILESUFFIX], test_name, '-V7.3')
\ No newline at end of file

Added: trunk/scipy/io/matlab/tests/test_byteordercodes.py
===================================================================
--- trunk/scipy/io/matlab/tests/test_byteordercodes.py	2008-10-05 03:36:54 UTC (rev 4774)
+++ trunk/scipy/io/matlab/tests/test_byteordercodes.py	2008-10-05 04:49:23 UTC (rev 4775)
@@ -0,0 +1,30 @@
+''' Tests for byteorder module '''
+
+import sys
+
+import numpy as np
+
+from numpy.testing import assert_raises
+
+import scipy.io.byteordercodes as sibc
+
+def test_native():
+    native_is_le = sys.byteorder == 'little'
+    assert sibc.sys_is_le == native_is_le
+    
+def test_to_numpy():
+    if sys.byteorder == 'little':
+        assert sibc.to_numpy_code('native') == '<'
+        assert sibc.to_numpy_code('swapped') == '>'
+    else:
+        assert sibc.to_numpy_code('native') == '>' 
+        assert sibc.to_numpy_code('swapped') == '<'
+    assert sibc.to_numpy_code('native') == sibc.to_numpy_code('=')
+    assert sibc.to_numpy_code('big') == '>'
+    for code in ('little', '<', 'l', 'L', 'le'):
+        assert sibc.to_numpy_code(code) == '<'
+    for code in ('big', '>', 'b', 'B', 'be'):
+        assert sibc.to_numpy_code(code) == '>'
+    assert_raises(ValueError, sibc.to_numpy_code, 'silly string')
+   
+

Modified: trunk/scipy/io/matlab/tests/test_mio.py
===================================================================
--- trunk/scipy/io/matlab/tests/test_mio.py	2008-10-05 03:36:54 UTC (rev 4774)
+++ trunk/scipy/io/matlab/tests/test_mio.py	2008-10-05 04:49:23 UTC (rev 4775)
@@ -17,7 +17,7 @@
 
 test_data_path = os.path.join(os.path.dirname(__file__), 'data')
 
-def _check_level(self, label, expected, actual):
+def _check_level(label, expected, actual):
     """ Check one level of a potentially nested object / list """
     # object array is returned from cell array in mat file
     typex = type(expected)
@@ -27,7 +27,7 @@
         assert len(expected) == len(actual), "Different list lengths at %s" % label
         for i, ev in enumerate(expected):
             level_label = "%s, [%d], " % (label, i)
-            self._check_level(level_label, ev, actual[i])
+            _check_level(level_label, ev, actual[i])
         return
     # object, as container for matlab structs and objects
     elif isinstance(expected, MatlabObject):
@@ -42,7 +42,7 @@
             ev = expected.__dict__[k]
             v = actual.__dict__[k]
             level_label = "%s, property %s, " % (label, k)
-            self._check_level(level_label, ev, v)
+            _check_level(level_label, ev, v)
         return
     # hoping this is a single value, which might be an array
     if SP.issparse(expected):
@@ -60,34 +60,24 @@
                "Types %s and %s do not match at %s" % (typex, typac, label)
         assert_equal(actual, expected, err_msg=label)
 
-def _check_case(self, name, files, case):
+def _check_case(name, files, case, *args, **kwargs):
     for file_name in files:
-        matdict = loadmat(file_name, struct_as_record=True)
+        matdict = loadmat(file_name, *args, **kwargs)
         label = "test %s; file %s" % (name, file_name)
         for k, expected in case.items():
             k_label = "%s, variable %s" % (label, k)
             assert k in matdict, "Missing key at %s" % k_label
-            self._check_level(k_label, expected, matdict[k])
+            _check_level(k_label, expected, matdict[k])
 
-# Add the load tests dynamically, with given parameters
-def _make_check_case(name, files, expected):
-    def cc(self):
-        self._check_case(name, files, expected)
-    cc.__doc__ = "check loadmat case %s" % name
-    return cc
+# Round trip tests 
+def _rt_check_case(name, expected, format):
+    mat_stream = StringIO()
+    savemat(mat_stream, expected, format=format)
+    mat_stream.seek(0)
+    _check_case(name, [mat_stream], expected, struct_as_record=True)
 
-# Add the round trip tests dynamically, with given parameters
-def _make_rt_check_case(name, expected, format):
-    def cc(self):
-        mat_stream = StringIO()
-        savemat(mat_stream, expected, format=format)
-        mat_stream.seek(0)
-        self._check_case(name, [mat_stream], expected)
-    cc.__doc__ = "check loadmat case %s" % name
-    return cc
-
 # Define cases to test
-theta = pi/4*arange(9,dtype=float)
+theta = pi/4*arange(9,dtype=float).reshape(9,1)
 case_table4 = [
     {'name': 'double',
      'expected': {'testdouble': theta}
@@ -224,7 +214,7 @@
         filt = os.path.join(test_data_path, 'test%s_*.mat' % name)
         files = glob(filt)
         assert files, "No files for test %s using filter %s" % (name, filt)
-        yield _make_check_case, name, files, expected
+        yield _check_case, name, files, expected
 
     # round trip tests
 def test_round_trip():
@@ -232,7 +222,7 @@
         name = case['name'] + '_round_trip'
         expected = case['expected']
         format = case in case_table4 and '4' or '5'
-        yield _make_rt_check_case, name, expected, format
+        yield _rt_check_case, name, expected, format
 
 def test_gzip_simple():
     xdense = zeros((20,20))
@@ -261,3 +251,9 @@
                               expected['x'].todense())
 
     
+def test_mat73():
+    # Check any hdf5 files raise an error
+    filenames = glob(
+        os.path.join(test_data_path, 'testhdf5*.mat'))
+    for filename in filenames:
+        assert_raises(NotImplementedError, loadmat, filename, struct_as_record=True)



More information about the Scipy-svn mailing list