[Scipy-svn] r2236 - trunk/Lib/io

scipy-svn at scipy.org scipy-svn at scipy.org
Fri Sep 29 06:36:40 CDT 2006


Author: matthew.brett at gmail.com
Date: 2006-09-29 06:36:34 -0500 (Fri, 29 Sep 2006)
New Revision: 2236

Removed:
   trunk/Lib/io/bytestream.py
Modified:
   trunk/Lib/io/mio4.py
   trunk/Lib/io/mio5.py
   trunk/Lib/io/miobase.py
Log:
Cleanups to processor function, drafting mat5 writing

Deleted: trunk/Lib/io/bytestream.py
===================================================================
--- trunk/Lib/io/bytestream.py	2006-09-26 16:59:28 UTC (rev 2235)
+++ trunk/Lib/io/bytestream.py	2006-09-29 11:36:34 UTC (rev 2236)
@@ -1,69 +0,0 @@
-# Author: Matthew Brett
-
-''' File-like interfact for memmapped array '''
-
-from numpy import *
-
-class ByteStream(object):
-    ''' Overlays file-like interface on memmapped array
-
-    This may speed up array reading from files
-    
-    @byte_array         - uint array or string containing bytes
-    '''
-
-    def __init__(self, byte_array):
-        if isinstance(byte_array, ndarray):
-            if not byte_array.dtype == uint8:
-                raise ValueError, 'Need uint8 byte array as array input'
-            self.bytes = byte_array
-        elif isinstance(byte_array, basestring):
-            self.bytes = ndarray(
-                shape=(len(byte_array)),
-                dtype=uint8,
-                buffer=byte_array)
-        else:
-            raise ValueError, "Need string or byte array as input"
-        self.array_len = len(byte_array)
-        self.seek(0)
-
-    # current file position
-    def get_pos(self):
-        return self._pos
-    def set_pos(self, offset):
-        if offset < 0:
-            raise IOError, 'Invalid argument'
-        self._pos = offset
-    pos = property(get_pos,
-                   set_pos,
-                   None,
-                   'get/set current position')
-    
-    def seek(self, offset, whence=0):
-        """ Method emulates seek method of file objects """
-        if whence == 0:
-            self.pos = offset
-        elif whence == 1: # seek relative to the current position
-            self.pos += offset
-        elif whence == 2: # relative to end
-            self.pos = self.array_len + offset
-        else:
-            raise ValueError, 'Invalid value %d for whence parameter' % whence
-
-    def tell(self):
-        return self.pos
-    
-    def read(self, num_bytes=-1):
-        if num_bytes < 0:
-            num_bytes = self.array_len
-        if self.pos >= self.array_len:
-            return array([], dtype=uint8)
-        next_pos = min(self.pos + num_bytes, self.array_len)
-        res = self.bytes[self.pos:next_pos]
-        self.pos = next_pos
-        return res
-
-    def write(self, data):
-        assert False, 'Not implemented'
-        
-

Modified: trunk/Lib/io/mio4.py
===================================================================
--- trunk/Lib/io/mio4.py	2006-09-26 16:59:28 UTC (rev 2235)
+++ trunk/Lib/io/mio4.py	2006-09-29 11:36:34 UTC (rev 2236)
@@ -323,9 +323,6 @@
     arr = array(arr)
     if arr.dtype.hasobject:
         raise TypeError, 'Cannot save object arrays in Mat4'
-    if have_sparse:
-        if scipy.sparse.issparse(arr):
-            return Mat4SparseWriter(stream, arr, name)
     if arr.dtype.kind in ('U', 'S'):
         return Mat4CharWriter(stream, arr, name)
     else:

Modified: trunk/Lib/io/mio5.py
===================================================================
--- trunk/Lib/io/mio5.py	2006-09-26 16:59:28 UTC (rev 2235)
+++ trunk/Lib/io/mio5.py	2006-09-29 11:36:34 UTC (rev 2236)
@@ -517,5 +517,193 @@
         return 0 not in mopt_bytes
 
 
-class Mat5Writer(MatFileWriter):
-    pass
+class Mat5MatrixWriter(MatStreamWriter):
+
+    def write_header(self, mclass,
+                     is_global,
+                     is_complex=False,
+                     is_logical=False,
+                     nzmax=0):
+        ''' Write header for given data options
+        @mclass      - mat5 matrix class
+        @is_global   - True if matrix is global
+        @is_complex  - True is matrix is complex
+        @is_logical  - True if matrix is logical
+        '''
+        dims = self.arr.shape
+        header = empty((), mdtypes_template['header'])
+        M = not ByteOrder.little_endian
+        O = 0
+        header['mopt'] = (M * 1000 +
+                          O * 100 + 
+                          P * 10 +
+                          T)
+        header['mrows'] = dims[0]
+        header['ncols'] = dims[1]
+        header['imagf'] = imagf
+        header['namlen'] = len(self.name) + 1
+        self.write_bytes(header)
+        self.write_string(self.name + '\0')
+        
+    def write(self):
+        assert False, 'Not implemented'
+
+
+class Mat5NumericWriter(Mat5MatrixWriter):
+
+    def write(self):
+        # identify matlab type for array
+        # make at least 2d
+        # write miMATRIX tag
+        # write array flags (complex, global, logical, class, nzmax)
+        # dimensions
+        # array name
+        # maybe downcast array to smaller matlab type
+        # write real
+        # write imaginary
+        # put padded length in miMATRIX tag
+        pass
+    
+
+class Mat5CharWriter(Mat5MatrixWriter):
+
+    def write(self):
+        self.arr_to_chars()
+        self.arr_to_2d()
+        dims = self.arr.shape
+        self.write_header(P=miUINT8,
+                          T=mxCHAR_CLASS)
+        if self.arr.dtype.kind == 'U':
+            # Recode unicode to ascii
+            n_chars = product(dims)
+            st_arr = ndarray(shape=(),
+                             dtype=self.arr_dtype_number(n_chars),
+                             buffer=self.arr)
+            st = st_arr.item().encode('ascii')
+            self.arr = ndarray(shape=dims, dtype='S1', buffer=st)
+        self.write_bytes(self.arr)
+
+
+class Mat5SparseWriter(Mat5MatrixWriter):
+
+    def write(self):
+        ''' Sparse matrices are 2D
+        See docstring for Mat5SparseGetter
+        '''
+        imagf = self.arr.dtype.kind == 'c'
+        N = self.arr.nnz
+        ijd = zeros((N+1, 3+imagf), dtype='f8')
+        for i in range(N):
+            ijd[i,0], ijd[i,1] = self.arr.rowcol(i)
+        ijd[:-1,0:2] += 1 # 1 based indexing
+        if imagf:
+            ijd[:-1,2] = self.arr.data.real
+            ijd[:-1,3] = self.arr.data.imag
+        else:
+            ijd[:-1,2] = self.arr.data
+        ijd[-1,0:2] = self.arr.shape
+        self.write_header(P=miDOUBLE,
+                          T=mxSPARSE_CLASS,
+                          dims=ijd.shape)
+        self.write_bytes(ijd)
+        
+    
+def matrix_writer_factory(stream, arr, name, unicode_strings=False, is_global=False):
+    ''' Factory function to return matrix writer given variable to write
+    @stream      - file or file-like stream to write to
+    @arr         - array to write
+    @name        - name in matlab (TM) workspace
+    '''
+    if have_sparse:
+        if scipy.sparse.issparse(arr):
+            return Mat5SparseWriter(stream, arr, name, is_global)
+    arr = array(arr)
+    if arr.dtype.hasobject:
+        types, arr_type = classify_mobjects(arr)
+        if arr_type == 'c':
+            return Mat5CellWriter(stream, arr, name, is_global, types)
+        elif arr_type == 's':
+            return Mat5StructWriter(stream, arr, name, is_global)
+        elif arr_type == 'o':
+            return Mat5ObjectWriter(stream, arr, name, is_global)
+    if arr.dtype.kind in ('U', 'S'):
+        if unicode_strings:
+            return Mat5UniCharWriter(stream, arr, name, is_global)
+        else:
+            return Mat5IntCharWriter(stream, arr, name, is_global)            
+    else:
+        return Mat5NumericWriter(stream, arr, name, is_global)
+                    
+def classify_mobjects(objarr):
+    ''' Function to classify objects passed for writing
+    returns
+    types         - S1 array of same shape as objarr with codes for each object
+                    i  - invalid object
+                    a  - ndarray
+                    s  - matlab struct
+                    o  - matlab object
+    arr_type       - one of
+                    c  - cell array
+                    s  - struct array
+                    o  - object array
+    '''
+    N = objarr.size
+    types = empty((N,), dtype='S1')
+    types[:] = 'i'
+    type_set = set()
+    flato = objarr.flat
+    for i in range(N):
+        obj = flato[i]
+        if isinstance(obj, ndarray):
+            types[i] = 'a'
+            continue
+        try:
+            fns = tuple(obj._fieldnames)
+        except AttributeError:
+            continue
+        try:
+            cn = obj._classname
+        except AttributeError:
+            types[i] = 's'
+            type_set.add(fns)
+            continue
+        types[i] = 'o'
+        type_set.add((cn, fns))
+    arr_type = 'c'
+    if len(set(types))==1 and len(type_set) == 1:
+        arr_type = types[0]
+    return types.reshape(objarr.shape), arr_type
+           
+        
+class MatFile5Writer(MatFileWriter):
+    ''' Class for writing mat5 files '''
+    def __init__(self, file_stream,
+                 do_compression=False,
+                 unicode_strings=False,
+                 global_vars=None):
+        super(MatFile5Writer, self).__init__(file_stream)
+        self.do_compression = do_compression
+        self.unicode_strings = unicode_strings
+        if global_vars:
+            self.global_vars = global_vars
+        else:
+            self.global_vars = []
+        
+    def put_variables(self, mdict):
+        for name, var in mdict.items():
+            is_global = name in self.global_vars
+            stream = StringIO()
+            matrix_writer_factory(stream,
+                                  var,
+                                  name,
+                                  is_global,
+                                  self.unicode_strings,
+                                  ).write()
+            if self.do_compression:
+                str = zlib.compress(stream.getvalue())
+                tag = empty((), mdtypes_template['tag_full'])
+                tag['mdtype'] = miCOMPRESSED
+                tag['byte_count'] = len(str)
+                self.file_stream.write(tag.tostring() + str)
+            else:
+                self.file_stream.write(stream.getvalue())

Modified: trunk/Lib/io/miobase.py
===================================================================
--- trunk/Lib/io/miobase.py	2006-09-26 16:59:28 UTC (rev 2235)
+++ trunk/Lib/io/miobase.py	2006-09-29 11:36:34 UTC (rev 2236)
@@ -88,8 +88,9 @@
                           in ('native', '=')
                           or in ('little', '<')
                           or in ('BIG', '>')
-    @base_name          - base name for unnamed variables
-    @matlab_compatible  - return arrays as matlab (TM) saved them
+    @base_name          - base name for unnamed variables (unused in code)
+    @mat_dtype          - return arrays in same dtype as loaded into matlab
+                          (instead of the dtype with which they are saved)
     @squeeze_me         - whether to squeeze unit dimensions or not
     @chars_as_strings   - whether to convert char arrays to string arrays
 
@@ -106,7 +107,7 @@
     def __init__(self, mat_stream,
                  byte_order=None,
                  base_name='raw',
-                 matlab_compatible=False,
+                 mat_dtype=False,
                  squeeze_me=True,
                  chars_as_strings=True,
                  ):
@@ -119,22 +120,26 @@
         self.base_name = base_name
         self._squeeze_me = squeeze_me
         self._chars_as_strings = chars_as_strings
-        self.matlab_compatible = matlab_compatible
+        self._mat_dtype = mat_dtype
+        self.processor_func = self.get_processor_func()
         
-    # matlab_compatible property sets squeeze_me and chars_as_strings
-    def get_matlab_compatible(self):
-        return self._matlab_compatible
-    def set_matlab_compatible(self, m_l_c):
-        self._matlab_compatible = m_l_c
-        if m_l_c:
-            self._squeeze_me = False
-            self._chars_as_strings = False
+    def set_matlab_compatible(self):
+        ''' Sets options to return arrays as matlab (tm) loads them '''
+        self._mat_dtype = True
+        self._squeeze_me = False
+        self._chars_as_strings = False
         self.processor_func = self.get_processor_func()
-    matlab_compatible = property(get_matlab_compatible,
-                                 set_matlab_compatible,
-                                 None,
-                                 'get/set matlab_compatible property')
 
+    def get_mat_dtype(self):
+        return self._mat_dtype
+    def set_mat_dtype(self, mat_dtype):
+        self._mat_dtype = mat_dtype
+        self.processor_func = self.get_processor_func()
+    mat_dtype = property(get_mat_dtype,
+                         set_mat_dtype,
+                         None,
+                         'get/set mat_dtype property')
+
     def get_squeeze_me(self):
         return self._squeeze_me
     def set_squeeze_me(self, squeeze_me):
@@ -151,9 +156,9 @@
         self._chars_as_strings = chars_as_strings
         self.processor_func = self.get_processor_func()
     chars_as_strings = property(get_chars_as_strings,
-                          set_chars_as_strings,
-                          None,
-                          'get/set squeeze me property')
+                                set_chars_as_strings,
+                                None,
+                                'get/set squeeze me property')
     
     def get_order_code(self):
         return self._order_code
@@ -200,7 +205,7 @@
 
         The read array is the first argument.
         The getter, passed as second argument to the function, must
-        define properties, iff matlab_compatible option is True:
+        define properties, iff mat_dtype option is True:
         
         mat_dtype    - data type when loaded into matlab (tm)
                        (None for no conversion)
@@ -223,7 +228,7 @@
                         arr[...,i] = self.chars_to_str(str_arr[i])
                 else: # return string
                     arr = self.chars_to_str(arr)
-            if self.matlab_compatible:
+            if self.mat_dtype:
                 # Apply options to replicate matlab's (TM)
                 # load into workspace
                 if getter.mat_dtype is not None:



More information about the Scipy-svn mailing list