[Numpy-discussion] saving incrementally numpy arrays
Citi, Luca
lciti@essex.ac...
Tue Aug 11 15:26:44 CDT 2009
You can do something a bit tricky but possibly working.
I made the assumption of a C-ordered 1d vector.
import numpy as np
import numpy.lib.format as fmt
# example of chunks
chunks = [np.arange(l) for l in range(5,10)]
# at the beginning
fp = open('myfile.npy', 'wb')
d = dict(
descr=fmt.dtype_to_descr(chunks[0].dtype),
fortran_order=False,
shape=(2**30), # some big shape you think you'll never reach
)
fp.write(fmt.magic(1,0))
fmt.write_array_header_1_0(fp, d)
h_len = fp.tell()
l = 0
# ... for each chunk ...
for chunk in chunks:
l += len(chunk)
fp.write(chunk.tostring('C'))
# finally
fp.seek(0,0)
fp.write(fmt.magic(1,0))
d['shape'] = (l,)
fmt.write_array_header_1_0(fp, d)
fp.write(' ' * (h_len - fp.tell() - 1))
fp.close()
More information about the NumPy-Discussion
mailing list