[Scipy-svn] r2341 - in trunk/Lib/io: . tests

scipy-svn at scipy.org scipy-svn at scipy.org
Thu Nov 30 07:40:40 CST 2006


Author: matthew.brett at gmail.com
Date: 2006-11-30 07:40:34 -0600 (Thu, 30 Nov 2006)
New Revision: 2341

Added:
   trunk/Lib/io/tests/test_recaster.py
Modified:
   trunk/Lib/io/__init__.py
   trunk/Lib/io/recaster.py
Log:
Further recaster debugging, with unit tests

Modified: trunk/Lib/io/__init__.py
===================================================================
--- trunk/Lib/io/__init__.py	2006-11-30 06:16:27 UTC (rev 2340)
+++ trunk/Lib/io/__init__.py	2006-11-30 13:40:34 UTC (rev 2341)
@@ -8,6 +8,7 @@
 from numpyio import packbits, unpackbits, bswap, fread, fwrite, \
      convert_objectarray
 from mio import *
+from recaster import Recaster
 from array_import import *
 from data_store import *
 from pickler import *

Modified: trunk/Lib/io/recaster.py
===================================================================
--- trunk/Lib/io/recaster.py	2006-11-30 06:16:27 UTC (rev 2340)
+++ trunk/Lib/io/recaster.py	2006-11-30 13:40:34 UTC (rev 2341)
@@ -6,6 +6,42 @@
 
 from numpy import *
 
+def sctype_attributes():
+    ''' Return dictionary describing numpy scalar types '''
+    d_dict = {}
+    for sc_type in ('complex','float'):
+        t_list = sctypes[sc_type]
+        for T in t_list:
+            F = finfo(T)
+            dt = dtype(T)
+            d_dict[T] = {
+                'kind': dt.kind,
+                'size': dt.itemsize,
+                'max': F.max,
+                'min': F.min}
+    for T in sctypes['int']:
+        dt = dtype(T)
+        sz = dt.itemsize
+        bits = sz*8-1
+        end = 2**bits
+        d_dict[T] = {
+            'kind': dt.kind,
+            'size': sz,
+            'min': -end,
+            'max': end-1
+            }
+    for T in sctypes['uint']:
+        dt = dtype(T)
+        sz = dt.itemsize
+        bits = sz*8
+        end = 2**bits
+        d_dict[T] = {
+            'kind': dt.kind,
+            'size': sz,
+            'min': 0,
+            'max': end
+        }
+    return d_dict
 
 class Recaster(object):
     ''' Class to recast arrays to one of acceptable scalar types
@@ -28,6 +64,8 @@
                      'int': 'i', 'i': 'i',
                      'uint': 'u', 'u': 'u'}
 
+    _sctype_attributes = sctype_attributes()
+
     def __init__(self, sctype_list=None, sctype_tols=None):
         ''' Set types for which we are attempting to downcast
 
@@ -37,68 +75,28 @@
         sctype_tols  - dictionary key datatype, values rtol, tol
                      to specify tolerances for checking near equality in downcasting
         '''
-        sys_dict = self.system_sctype_dict()
         if sctype_list is None:
-            self.sctype_dict = sys_dict.copy()
-            sctype_list = self.sctype_dict.keys()
-        else:
-            D = {}
-            for k, v in sys_dict.items():
-                if k in sctype_list:
-                    D[k] = v
-            self.sctype_dict = D
+            sctype_list = self._sctype_attributes.keys()
         self.sctype_list = sctype_list
         self.sctype_tols = self.default_sctype_tols()
         if sctype_tols is not None:
-            self.sctype_tols.merge(sctype_tols)
+            self.sctype_tols.update(sctype_tols)
         # Cache sctype sizes, 
         self.sized_sctypes = {}
         for k in ('c', 'f', 'i', 'u'):
             self.sized_sctypes[k] = self.sctypes_by_size(k)
-        self.int_sctypes = [T for T in self.sctype_list if dtype(T).kind in ('i', 'u')]
-        self.all_int_sized_sctypes = []
+        # All integer sizes
+        self.ints_sized_sctypes = []
         for k, v in self.sized_sctypes.items():
             if k in ('u', 'i'):
-                self.all_int_sized_sctypes.append(v)
-        self.nearest_dtypes = {}
-        for k in sys_dict:
-            self.nearest_dtypes[k] = self._nearest_dtype(k)
-        self.capable_dtypes = {}
-        for k in sys_dict:
-            self.capable_dtypes[k] = self._capable_dtype(k)
-        
-    def system_sctype_dict(self):
-        d_dict = {}
-        for sc_type in ('complex','float'):
-            t_list = sctypes[sc_type]
-            for T in t_list:
-                dt = dtype(T)
-                d_dict[T] = {
-                    'kind': dt.kind,
-                    'size': dt.itemsize}
-        for T in sctypes['int']:
-            dt = dtype(T)
-            sz = dt.itemsize
-            bits = sz*8-1
-            end = 2**bits
-            d_dict[T] = {
-                'kind': dt.kind,
-                'size': sz,
-                'min': -end,
-                'max': end-1
-                }
-        for T in sctypes['uint']:
-            dt = dtype(T)
-            sz = dt.itemsize
-            bits = sz*8
-            end = 2**bits
-            d_dict[T] = {
-                'kind': dt.kind,
-                'size': sz,
-                'min': 0,
-                'max': end
-            }
-        return d_dict
+                for e in v:
+                    self.ints_sized_sctypes.append(e)
+        if self.ints_sized_sctypes:
+            self.ints_sized_sctypes.sort(lambda x, y: cmp(y[1], x[1]))
+        # Capable types list
+        self._capable_sctypes = {}
+        for k in self._sctype_attributes:
+            self._capable_sctypes[k] = self.get_capable_sctype(k)
     
     def default_sctype_tols(self):
         ''' Default allclose tolerance values for all dtypes '''
@@ -121,11 +119,6 @@
                     'atol': F.tiny}
         return t_dict
 
-    def tols_from_sctype(self, sctype):
-        ''' Return rtol and atol for sctype '''
-        tols = self.sctype_tols[sctype]
-        return tols['rtol'], tols['atol']
-        
     def sctypes_by_size(self, sctype):
         ''' Returns storage size ordered list of entries of scalar type sctype
 
@@ -144,35 +137,19 @@
                 D.append([t, dt.itemsize])
         D.sort(lambda x, y: cmp(y[1], x[1]))
         return D
-    
-    def _nearest_sctype(self, sct):
-        ''' Return scalar type closest in size to that of sct
 
-        Input
-        sct     - sctype
+    def capable_sctype(self, sct):
+        ''' Return smallest type containing sct type without precision loss
 
-        ID = input sctype. AT = acceptable sctype.  Return ID if ID is
-        in ATs. If ID is smaller / larger than all ATs, return
-        smallest / largest AT. Otherwise return nearest AT larger than
-        ID.
+        Value pulled fron dictionary cached from init - see
+        get_capable_sctype method for algorithm
         '''
-        dt = dtype(sct)
-        if sct in self.sctype_list:
-            return sct
-        sctypes = self.sized_sctypes[dt.kind]
-        if not sctypes:
+        try:
+            return self._capable_sctypes[sct]
+        except KeyError:
             return None
-        dti = sct.itemsize
-        for i, t in enumerate(sctypes):
-            if t[1] < dti:
-                break
-        else:
-            return t[0]
-        if i:
-            i-=1
-        return sctypes[i][0]
         
-    def _capable_sctype(self, sct):
+    def get_capable_sctype(self, sct):
         ''' Return smallest scalar type containing sct type without precision loss
 
         Input
@@ -194,33 +171,30 @@
         out_t = None
         # Unsigned and signed integers
         # Precision loss defined by max min outside datatype range
-        dt = dtype(sct)
-        if dt.kind in ('u', 'i'):
-            sctypes = self.all_int_sized_sctypes
-            if not sctypes:
-                return None
-            D = self.sctype_dict
-            mx = D[sct]['max']
-            mn = D[sct]['min']
-            for i, t in enumerate(sctypes):
-                this_sct = t[0]
-                this_d = D[dt]
-                if this_d['max'] >= mx and this_d['min'] <= mn:
-                    out_t = this_sct
+        D = self._sctype_attributes[sct]
+        if D['kind'] in ('u', 'i'):
+            out_t = self.smallest_int_sctype(D['max'], D['min'])
         else:
             # Complex and float types
             # Precision loss defined by data size < sct
-            sctypes = self.sized_sctypes[sct]
+            sctypes = self.sized_sctypes[D['kind']]
             if not sctypes:
                 return None
-            dti = dtype(sct).itemsize
+            dti = D['size']
             out_t = None
             for i, t in enumerate(sctypes):
                 if t[1] >= dti:
                     out_t = t[0]
+                else:
+                    break
         return out_t
 
-    def smaller_same_kind(self, arr):
+    def tols_from_sctype(self, sctype):
+        ''' Return rtol and atol for sctype '''
+        tols = self.sctype_tols[sctype]
+        return tols['rtol'], tols['atol']
+        
+    def smallest_same_kind(self, arr):
         ''' Return arr maybe downcast to same kind, smaller storage
 
         If arr cannot be downcast within given tolerances, then:
@@ -228,10 +202,10 @@
         return None
         '''
         dtp = arr.dtype
-        dti = dt.itemsize
+        dti = dtp.itemsize
         sctypes = self.sized_sctypes[dtp.kind]
         sctypes = [t[0] for i, t in enumerate(sctypes) if t[1] < dti]
-        return self.smaller_from_sctypes(arr, sctypes)
+        return self.smallest_from_sctypes(arr, sctypes)
 
     def smallest_from_sctypes(self, arr, sctypes):
         ''' Returns array recast to smallest possible type from list
@@ -249,14 +223,14 @@
             test_arr = arr.astype(T)
             if allclose(test_arr, arr, rtol, atol):
                 ret_arr = test_arr
+                can_downcast = True
             else:
                 break
-        else: # No downcasting withing tolerance
-            if dt not in self.sctype_list:
-                return None
+        if ret_arr.dtype.type not in self.sctype_list:
+            return None
         return ret_arr
         
-    def smallest_int_dtype(self, mx, mn):
+    def smallest_int_sctype(self, mx, mn):
         ''' Return integer type with smallest storage containing mx and mn
 
         Inputs
@@ -265,15 +239,14 @@
 
         Returns None if no integer can contain this range
         '''
-        dt = None
-        for T in self.int_sctypes:
-            t_dict = self.sctype_dict[T]
+        sct = None
+        for T, tsz in self.ints_sized_sctypes:
+            t_dict = self._sctype_attributes[T]
             if t_dict['max'] >= mx and t_dict['min'] <= mn:
-                c_sz = t_dict['size']
-                if dt is None or c_sz < sz:
-                    dt = T
-                    sz = c_sz
-        return dt
+                if sct is None or tsz < sz:
+                    sct = T
+                    sz = tsz
+        return sct
 
     def recast(self, arr):
         ''' Try arr downcast, upcast if necesary to get compatible type '''
@@ -317,7 +290,7 @@
         if allclose(arr, test_arr, rtol, atol):
             return self.downcast_float(test_arr)
         # try downcasting to another complex type
-        return self.smaller_same_kind(arr)
+        return self.smallest_same_kind(arr)
     
     def downcast_float(self, arr):
         # Try integer
@@ -326,7 +299,7 @@
         if allclose(arr, test_arr, rtol, atol):
             return test_arr
         # Otherwise descend the float types
-        return self.smaller_same_kind(arr)
+        return self.smallest_same_kind(arr)
 
     def downcast_integer(self, arr):
         ''' Downcasts arr to integer
@@ -336,7 +309,7 @@
         '''
         mx = amax(arr)
         mn = amin(arr)
-        idt = self.smallest_int_dtype(mx, mn)
+        idt = self.smallest_int_sctype(mx, mn)
         if idt:
             return arr.astype(idt)
         return None

Added: trunk/Lib/io/tests/test_recaster.py
===================================================================
--- trunk/Lib/io/tests/test_recaster.py	2006-11-30 06:16:27 UTC (rev 2340)
+++ trunk/Lib/io/tests/test_recaster.py	2006-11-30 13:40:34 UTC (rev 2341)
@@ -0,0 +1,45 @@
+from numpy.testing import *
+import numpy as N
+
+set_package_path()
+from io.recaster import sctype_attributes, Recaster
+restore_path()
+
+try:  # Python 2.3 support
+    from sets import Set as set
+except:
+    pass
+
+class test_recaster(ScipyTestCase):
+    def setUp(self):
+        self.recaster = Recaster([N.int32, N.complex64, N.float32])
+    
+    def test_init(self):
+        # Setting sctype_list
+        R = Recaster()
+        assert set(R.sctype_list) == set(sctype_attributes().keys()), \
+                               'Default recaster should include all system types'
+        T = N.float32
+        R = Recaster([T])
+        assert R.sctype_list == [T], 'Scalar type list not correctly set'
+        # Setting tolerances
+        tols = self.recaster.default_sctype_tols()
+        assert tols == self.recaster.sctype_tols, 'Unexpected tols dictionary'
+        F = N.finfo(T)
+        R = Recaster(sctype_tols={T: {'rtol': F.eps*2, 'atol': F.tiny*2, 'silly': 'silly text'}})
+        assert tols != R.sctype_tols, 'Tols dictionary not set correctly'
+        r, a = R.tols_from_sctype(T)
+        assert r == F.eps*2, 'Rtol not correctly set'
+        assert a == F.tiny*2, 'Atol not correctly set'
+        # Sctype size lists
+        # Integer sizes
+        # Cabable types
+        
+    def test_methods(self):
+        A = N.array(1, N.float64)
+        B = A.astype(N.float32)
+        # smallest from sctypes
+        C = self.recaster.smallest_from_sctypes(A, [N.float32])
+        # smaller same kind
+        C = self.recaster.smallest_same_kind(A)
+        assert C.dtype == N.dtype(N.float32), 'Dtype was not downcast'



More information about the Scipy-svn mailing list