[Numpy-discussion] some recarray rework

Francesc Alted falted at openlc.org
Wed Jan 8 13:27:06 CST 2003


In the context of optimizing the PyTables support for numarray and recarray
objects I have been playing with recarray module, and ended with a
somewhat improved version of it. Roughly, the modifications done are:

- Addition of a cache to quickly access the columns (numarrays) in
  recarrays. This object is a map (dictionary) where keys are the name
  fields and values are the pointers to columns regarded as numarrays
  entities. This dictionary is accessible through the new attribute

- Addition of an attribute for recarray objects named "_record" which
  points to a special object ("Record2" class) and that it is aware of
  the "_fields" cache. It that can be used to access the different
  rows in recarray objects in an efficient way.

- The "_record" object is callable (it defines the "__call__" method)
  so as to select the recarray row that is active during access to the
  different fields.


- Access to rows and columns (fields) in recarray objects are one
  order of magnitude faster (!).

- The new "_fields" and "_record" attributes provides convenient and
  intuitive ways to access the information in recarrays.

- The "_record" attribute suports the "__getattr__" and "__setattr__"
  methods that are very convenient to access fields in a row.


- "_record" attribute points always to the same object and you must
  pass it the row over which you want to operate. So, if you want to
  have two different objects pointing to different rows, you can't use
  the "_record" attribute to get them (but you can still use the
  existing Record class through by calling the "__getitem__" method
  of a recarray object).

- Two new attributes are added to the already large number of recarray
  variables. However, this new variables has no special space
  requirements as "_record" object has only three scalar variables
  and "_fields" is a dictionary with many entries as fields in
  recarray, which should be not a large amount.

I'm attaching this modified version as well as a testbed program in order to
test their new access methods and improved performance. The output of this
program ran in a pentium4 at 2GHz machine is also included.

Feel free to play with it and/or take/adapt the parts you consider better
suited to recarray module.

Francesc Alted                            PGP KeyID:      0x61C8C11F
-------------- next part --------------
import numarray as num
import ndarray as mda
import memory
import chararray
import sys, copy, os, re, types, string

__version__ = '1.0'

class Char:
    """ data type Char class"""
    bytes = 1
    def __repr__(self):
        return "CharType"

CharType = Char()

# translation table to the num data types
numfmt = {'i1':num.Int8, 'u1':num.UInt8, 'i2':num.Int16, 'i4':num.Int32,
          'f4':num.Float32, 'f8':num.Float64,
          'l':num.Bool, 'b':num.Int8, 'u':num.UInt8, 's':num.Int16,
          'i':num.Int32, 'N':num.Int64,
          'f':num.Float32, 'd':num.Float64, 'r':num.Float32,
          'Int8':num.Int8, 'Int16':num.Int16, 'Int32':num.Int32,
          'UInt8':num.UInt8, 'Float32':num.Float32, 'Float64':num.Float64,

# the reverse translation table of the above (for numarray only)
revfmt = {num.Int16:'s', num.Int32:'i', num.Int64:'N',
          num.Float32:'r', num.Float64:'d',
          num.Bool:'l', num.Int8:'b', num.UInt8:'u', CharType:'a'}

# TFORM regular expression
format_re = re.compile(r'(?P<repeat>^[0-9]*)(?P<dtype>[A-Za-z0-9.]+)')

def fromrecords (recList, formats=None, names=None):
    """ create a Record Array from a list of records in text form

        The data in the same field can be heterogeneous, they will be promoted
        to the highest data type.  This method is intended for creating
        smaller record arrays.  If used to create large array e.g.


        it is slow.

    >>> r=fromrecords([[456,'dbe',1.2],[2,'de',1.3]],names='col1,col2,col3')
    >>> print r[0]
    (456, 'dbe', 1.2)
    >>> r.field('col1')
    array([456,   2])
    >>> r.field('col2')
    CharArray(['dbe', 'de'])
    >>> import cPickle
    >>> print cPickle.loads(cPickle.dumps(r))
    (456, 'dbe', 1.2),
    (2, 'de', 1.3)

    _shape = len(recList)
    _nfields = len(recList[0])
    for _rec in recList:
        if len(_rec) != _nfields:
            raise ValueError, "inconsistent number of objects in each record"
    arrlist = [0]*_nfields
    for col in range(_nfields):
        tmp = [0]*_shape
        for row in range(_shape):
            tmp[row] = recList[row][col]
            arrlist[col] = num.array(tmp)
                arrlist[col] = chararray.array(tmp)
                raise ValueError, "inconsistent data at row %d,field %d" % (row, col)
    _array = fromarrays(arrlist, formats=formats, names=names)
    del arrlist
    del tmp
    return _array

def fromarrays (arrayList, formats=None, names=None):
    """ create a Record Array from a list of num/char arrays

    >>> x1=num.array([1,2,3,4])
    >>> x2=chararray.array(['a','dd','xyz','12'])
    >>> x3=num.array([1.1,2,3,4])
    >>> r=fromarrays([x1,x2,x3],names='a,b,c')
    >>> print r[1]
    (2, 'dd', 2.0)
    >>> x1[1]=34
    >>> r.field('a')
    array([1, 2, 3, 4])

    _shape = len(arrayList[0])

    if formats == None:

        # go through each object in the list to see if it is a numarray or
        # chararray and determine the formats
        formats = ''
        for obj in arrayList:
            if isinstance(obj, chararray.CharArray):
                formats += `obj._itemsize` + 'a,'
            elif isinstance(obj, num.NumArray):
                if len(obj._shape) == 1: _repeat = ''
                elif len(obj._shape) == 2: _repeat = `obj._shape[1]`
                else: raise ValueError, "doesn't support numarray more than 2-D"

                formats += _repeat + revfmt[obj._type] + ','
                raise ValueError, "item in the array list must be numarray or chararray"

    for obj in arrayList:
        if len(obj) != _shape:
            raise ValueError, "array has different lengths"

    _array = RecArray(None, formats=formats, shape=_shape, names=names)

    # populate the record array (make a copy)
    for i in range(len(arrayList)):
            _array.field(_array._names[i])[:] = arrayList[i]
            print "Incorrect CharArray format %s, copy unsuccessful." % _array._formats[i]
    return _array

def fromstring (datastring, formats, shape=0, names=None):
    """ create a Record Array from binary data contained in a string"""
    _array = RecArray(chararray._stringToBuffer(datastring), formats, shape, names)
    if mda.product(_array._shape)*_array._itemsize > len(datastring):
        raise ValueError("Insufficient input data.")
    else: return _array

def fromfile(file, formats, shape=-1, names=None):
    """Create an array from binary file data

    If file is a string then that file is opened, else it is assumed
    to be a file object. No options at the moment, all file positioning
    must be done prior to this function call with a file object

    >>> import testdata, sys
    >>> fd=open(testdata.filename)
    >>> fd.seek(2880*2)
    >>> r=fromfile(fd, formats='d,i,5a', shape=3)
    >>> r._byteorder = "big"
    >>> print r[0]
    (5.1000000000000005, 61, 'abcde')
    >>> r._shape

    if isinstance(shape, types.IntType) or isinstance(shape, types.LongType):
        shape = (shape,)
    name = 0
    if isinstance(file, types.StringType):
        name = 1
        file = open(file, 'rb')
    size = os.path.getsize(file.name) - file.tell()

    dummy = array(None, formats=formats, shape=0)
    itemsize = dummy._itemsize

    if shape and itemsize:
        shapesize = mda.product(shape)*itemsize
        if shapesize < 0:
            shape = list(shape)
            shape[ shape.index(-1) ] = size / -shapesize
            shape = tuple(shape)

    nbytes = mda.product(shape)*itemsize

    if nbytes > size:
        raise ValueError(
                "Not enough bytes left in file for specified shape and type")

    # create the array
    _array = RecArray(None, formats=formats, shape=shape, names=names)
    nbytesread = memory.file_readinto(file, _array._data)
    if nbytesread != nbytes:
        raise IOError("Didn't read as many bytes as expected")
    if name:
    return _array

# The test below was factored out of "array" due to platform specific
# floating point formatted results:  e+020 vs. e+20
if sys.platform == "win32":
    _fnumber = "2.5984589414244182e+020"
    _fnumber = "2.5984589414244182e+20"

__test__ = {}
__test__["array_platform_test_workaround"] = """
        >>> r=array('a'*200,'r,3s,5a,i',3)
        >>> print r[0]
        (%(_fnumber)s, array([24929, 24929, 24929], type=Int16), 'aaaaa', 1633771873)
        >>> print r[1]
        (%(_fnumber)s, array([24929, 24929, 24929], type=Int16), 'aaaaa', 1633771873)
        """ % globals()
del _fnumber

def array(buffer=None, formats=None, shape=0, names=None):
    """This function will creates a new instance of a RecArray.

    buffer      specifies the source of the array's initialization data.
                buffer can be: RecArray, list of records in text, list of
                numarray/chararray, None, string, buffer.

    formats     specifies the fromat definitions of the array's records.

    shape       specifies the array dimensions.

    names       specifies the field names.

    >>> r=array([[456,'dbe',1.2],[2,'de',1.3]],names='col1,col2,col3')
    >>> print r[0]
    (456, 'dbe', 1.2)
    >>> r=array('a'*200,'r,3i,5a,s',3)
    >>> r._bytestride
    >>> r._names
    ['c1', 'c2', 'c3', 'c4']
    >>> r._repeats
    [1, 3, 5, 1]
    >>> r._shape

    if (buffer is None) and (formats is None):
        raise ValueError("Must define formats if buffer=None")
    elif buffer is None or isinstance(buffer, types.BufferType):
        return RecArray(buffer, formats=formats, shape=shape, names=names)
    elif isinstance(buffer, types.StringType):
        return fromstring(buffer, formats=formats, shape=shape, names=names)
    elif isinstance(buffer, types.ListType) or isinstance(buffer, types.TupleType):
        if isinstance(buffer[0], num.NumArray) or isinstance(buffer[0], chararray.CharArray):
            return fromarrays(buffer, formats=formats, names=names)
            return fromrecords(buffer, formats=formats, names=names)
    elif isinstance(buffer, RecArray):
        return buffer.copy()
    elif isinstance(buffer, types.FileType):
        return fromfile(buffer, formats=formats, shape=shape, names=names)
        raise ValueError("Unknown input type")

def _RecGetType(name):
    """Converts a type repr string into a type."""
    if name == "CharType":
        return CharType
        return num._getType(name)

class RecArray(mda.NDArray):
    """Record Array Class"""

    def __init__(self, buffer, formats, shape=0, names=None, byteoffset=0,
                 bytestride=None, byteorder=sys.byteorder, aligned=1):

        # names and formats can be either a string with components separated
        # by commas or a list of string values, e.g. ['i4', 'f4'] and 'i4,f4'
        # are equivalent formats


        itemsize = self._stops[-1] + 1

        if shape != None:
            if type(shape) in [types.IntType, types.LongType]: shape = (shape,)
            elif (type(shape) == types.TupleType and type(shape[0]) in [types.IntType, types.LongType]):
            else: raise NameError, "Illegal shape %s" % `shape`

        #XXX need to check shape*itemsize == len(buffer)?

        self._shape = shape
        mda.NDArray.__init__(self, self._shape, itemsize, buffer=buffer,
        self._byteorder = byteorder

        # Build the column arrays
        self._fields = self._get_fields()

        # Associate a record object for accessing values in each row
        # in a efficient way (i.e. without creating a new object each time)
        self._record = Record2(self)

    def _parseFormats(self, formats):
        """ Parse the field formats """

        if (type(formats) in [types.ListType, types.TupleType]):
            _fmt = formats[:]           ### make a copy
        elif (type(formats) == types.StringType):
            _fmt = string.split(formats, ',')
            raise NameError, "illegal input formats %s" % `formats`

        self._nfields = len(_fmt)
        self._repeats = [1] * self._nfields
        self._sizes = [0] * self._nfields
        self._stops = [0] * self._nfields

        # preserve the input for future reference
        self._formats = [''] * self._nfields

        sum = 0
        for i in range(self._nfields):

            # parse the formats into repeats and formats
                (_repeat, _dtype) = format_re.match(string.strip(_fmt[i])).groups()
            except: print 'format %s is not recognized' % _fmt[i]

            if _repeat == '': _repeat = 1
            else: _repeat = eval(_repeat)
            _fmt[i] = numfmt[_dtype]
            self._repeats[i] = _repeat

            self._sizes[i] = _fmt[i].bytes * _repeat
            sum += self._sizes[i]
            self._stops[i] = sum - 1

            # Unify the appearance of _format, independent of input formats
            self._formats[i] = `_repeat`+revfmt[_fmt[i]]

        self._fmt = _fmt

    def __getstate__(self):
        """returns pickled state dictionary for RecArray"""
        state = mda.NDArray.__getstate__(self)
        state["_fmt"] = map(repr, self._fmt)
        return state
    def __setstate__(self, state):
        mda.NDArray.__setstate__(self, state)
        self._fmt = map(_RecGetType, state["_fmt"])

    def _fieldNames(self, names=None):
        """convert input field names into a list and assign to the _names
        attribute """

        if (names):
            if (type(names) in [types.ListType, types.TupleType]):
            elif (type(names) == types.StringType):
                names = string.split(names, ',')
                raise NameError, "illegal input names %s" % `names`

            self._names = map(lambda n:string.strip(n), names)
        else: self._names = []

        # if the names are not specified, they will be assigned as "c1, c2,..."
        # if not enough names are specified, they will be assigned as "c[n+1],
        # c[n+2],..." etc. where n is the number of specified names..."
        self._names += map(lambda i: 'c'+`i`, range(len(self._names)+1,self._nfields+1))

    def _get_fields(self):
        """ get a dictionary with fields as numeric arrays """

        # Iterate over all the fields
        fields = {}
        for fieldName in self._names:
            # determine the offset within the record
            indx = index_of(self._names, fieldName)
            _start = self._stops[indx] - self._sizes[indx] + 1

            _shape = self._shape
            _type = self._fmt[indx]
            _buffer = self._data
            _offset = self._byteoffset + _start

            # don't use self._itemsize due to possible slicing
            _stride = self._strides[0]

            _order = self._byteorder

            if isinstance(_type, Char):
                arr = chararray.CharArray(buffer=_buffer, shape=_shape,
                          itemsize=self._repeats[indx], byteoffset=_offset,
                arr = num.NumArray(shape=_shape, type=_type, buffer=_buffer,
                          byteoffset=_offset, bytestride=_stride,
                          byteorder = _order)

                # modify the _shape and _strides for array elements
                if (self._repeats[indx] > 1):
                    arr._shape = self._shape + (self._repeats[indx],)
                    arr._strides = (self._strides[0], _type.bytes)

            # Put this array as a value in dictionary
            fields[fieldName] = arr

        return fields

    def field(self, fieldName):
        """ get the field data as a numeric array """

        return self._fields[fieldName]
    def info(self):
        """display instance's attributes (except _data)"""
        _attrList = dir(self)
        for attr in _attrList:
            print '%s = %s' % (attr, getattr(self,attr))

    def __str__(self):
        outstr = 'RecArray[ \n'
        for i in self:
            outstr += Record.__str__(i) + ',\n'
        return outstr[:-2] + '\n]'

    ### The followng  __getitem__ is not in the requirements
    ### and is here for experimental purposes
    def __getitem__(self, key):
        if type(key) == types.TupleType:
            if len(key) == 1:
                return mda.NDArray.__getitem__(self,key[0])
            elif len(key) == 2 and type(key[1]) == types.StringType:
                return mda.NDArray.__getitem__(self,key[0]).field(key[1])
                raise NameError, "Illegal key %s" % `key`
        return mda.NDArray.__getitem__(self,key)

    def _getitem(self, key):
        byteoffset = self._getByteOffset(key)
        row = (byteoffset - self._byteoffset) / self._strides[0]
        return Record(self, row)

    def _setitem(self, key, value):
        byteoffset = self._getByteOffset(key)
        row = (byteoffset - self._byteoffset) / self._strides[0]
        for i in range(self._nfields):
            self.field(self._names[i])[row] = value.field(self._names[i])

    def reshape(*value):
        print "Cannot reshape record array."

class Record2:
    """Record2 Class

    This class is similar to Record except for the fact that it is
    created and associated with a recarray in their creation
    time. When speed in traversing the recarray is required this
    approach is more convenient than create a new Record object for
    each row that is visited.


    def __init__(self, input):

        self.__dict__["_array"] = input
        self.__dict__["_fields"] = input._fields
        self.__dict__["_row"] = 0

    def __call__(self, row):
        """ set the row for this record object """
        if row < self._array.shape[0]:
            self.__dict__["_row"] = row
            return self
            return None

    def __getattr__(self, fieldName):
        """ get the field data of the record"""
            return self._fields[fieldName][self._row]
            (type, value, traceback) = sys.exc_info()
            raise AttributeError, "Error accessing \"%s\" attr.\n %s" % \
                  (fieldName, "Error was: \"%s: %s\"" % (type,value))

    def __setattr__(self, fieldName, value):
        """ set the field data of the record"""

        self._fields[fieldName][self._row] = value

    def __str__(self):
        """ represent the record as an string """
        outlist = []
        for name in self._array._names:
        return "(" + ", ".join(outlist) + ")"

class Record:
    """Record Class"""

    def __init__(self, input, row=0):
        if isinstance(input, types.ListType) or isinstance(input, types.TupleType):
            input = fromrecords([input])
        if isinstance(input, RecArray):
            self.array = input
            self.row = row

    def __getattr__(self, fieldName):
        """ get the field data of the record"""

        #return self.array.field(fieldName)[self.row]
        if fieldName in self.array._names:
            #return self.array.field(fieldName)[self.row]
            return self.array._fields[fieldName][self.row]

    def field(self, fieldName):
        """ get the field data of the record"""

        #return self.array.field(fieldName)[self.row]
        return self.array.field(fieldName)[self.row]

    def __str__(self):
        outstr = '('
        #for i in range(self.array._nfields):
        #    print self.array.field(i)[self.row]
        for name in self.array._names:
            #print self.array.field(name)[self.row]
            #print self.array._fields[name][self.row]
            ### this is not efficient, need to know how to convert N-bytes to each data type
            outstr += `self.array.field(name)[self.row]` + ', '
        return outstr[:-2] + ')'

def index_of(nameList, key):
    """ Get the index of the key in the name list.

        The key can be an integer or string.  If integer, it is the index
        in the list.  If string, the name matching will be case-insensitive and
        trailing blank-insensitive.
    if (type(key) in [types.IntType, types.LongType]):
        indx = key
    elif (type(key) == types.StringType):
        _names = nameList[:]
        for i in range(len(_names)):
            _names[i] = string.lower(_names[i])
            indx = _names.index(string.strip(string.lower(key)))
            raise NameError, "Key %s does not exist" % key
        raise NameError, "Illegal key %s" % `key`

    return indx

def find_duplicate (list):
    """Find duplication in a list, return a list of dupicated elements"""
    dup = []
    for i in range(len(list)):
        if (list[i] in list[i+1:]):
            if (list[i] not in dup):
    return dup

def test():
    import doctest, recarray
    return doctest.testmod(recarray)

if __name__ == "__main__":
-------------- next part --------------
import sys, time
import numarray as num
import chararray
import recarray
import recarray2  # This is my modified version

usage = \
"""usage: %s recordlength
     Set recordlength to 1000 at least to obtain decent figures!
""" % sys.argv[0]

    reclen = int(sys.argv[1])
    print usage

delta = 0.000001

# Creation of recarrays objects for test
x2=chararray.array(None, itemsize=7, shape=reclen)
x3=num.array(num.arange(reclen,reclen*3,2), num.Float64)

print "recarray shape in test ==>", r2.shape

print "Assignment in recarray modified"
print "-------------------------------"
t1 = time.clock()
for row in xrange(reclen):
    rec = r2._record(row)  # select the row to be changed
    #rec.b = "changed"      # change the "b" field
    rec.c = float(row**2)  # Change the "c" field
t2 = time.clock()
ttime = round(t2-t1, 3)
print "Assign time:", ttime, " Rows/s:", int(reclen/(ttime+delta))
print "Field b on row 2 after re-assign:", r2.field("c")[2]

print "Assignment in recarray original"
print "-------------------------------"
t1 = time.clock()
for row in xrange(reclen):
    #r1.field("b")[row] = "changed"
    r1.field("c")[row] = float(row**2)
t2 = time.clock()
ttime = round(t2-t1, 3)
print "Assign time:", ttime, " Rows/s:", int(reclen/(ttime+delta))
print "Field b on row 2 after re-assign:", r1.field("c")[2]

print "Selection in recarray modified"
print "------------------------------"
t1 = time.clock()
for row in xrange(reclen):
    rec = r2._record(row)
    if rec.a < 3:
        print "This record pass the cut ==>", rec.c, "(row", row, ")"
t2 = time.clock()
ttime = round(t2-t1, 3)
print "Select time:", ttime, " Rows/s:", int(reclen/(ttime+delta))

print "Selection in recarray original"
print "------------------------------"
t1 = time.clock()
for row in xrange(reclen):
    rec = r1[row]
    if rec.field("a") < 3:
        print "This record pass the cut ==>", rec.field("c"), "(row", row, ")"
t2 = time.clock()
ttime = round(t2-t1, 3)
print "Select time:", ttime, " Rows/s:", int(reclen/(ttime+delta))

-------------- next part --------------
recarray shape in test ==> (10000,)
Assignment in recarray modified
Assign time: 0.15  Rows/s: 66666
Field b on row 2 after re-assign: 4.0

Assignment in recarray original
Assign time: 1.24  Rows/s: 8064
Field b on row 2 after re-assign: 4.0

Selection in recarray modified
This record pass the cut ==> 0.0 (row 0 )
This record pass the cut ==> 1.0 (row 1 )
This record pass the cut ==> 4.0 (row 2 )
Select time: 0.18  Rows/s: 55555

Selection in recarray original
This record pass the cut ==> 0.0 (row 0 )
This record pass the cut ==> 1.0 (row 1 )
This record pass the cut ==> 4.0 (row 2 )
Select time: 1.52  Rows/s: 6578

More information about the Numpy-discussion mailing list