[Numpy-discussion] saving incrementally numpy arrays

[Citi, Luca]

You can do something a bit tricky but possibly working.
I made the assumption of a C-ordered 1d vector.



import numpy as np
import numpy.lib.format as fmt

# example of chunks
chunks = [np.arange(l) for l in range(5,10)]

# at the beginning
fp = open('myfile.npy', 'wb')
d = dict(
            descr=fmt.dtype_to_descr(chunks[0].dtype),
            fortran_order=False,
            shape=(2**30), # some big shape you think you'll never reach
        )
fp.write(fmt.magic(1,0))
fmt.write_array_header_1_0(fp, d)
h_len = fp.tell()
l = 0
# ... for each chunk ...
for chunk in chunks:
    l += len(chunk)
    fp.write(chunk.tostring('C'))
# finally
fp.seek(0,0)
fp.write(fmt.magic(1,0))
d['shape'] = (l,)
fmt.write_array_header_1_0(fp, d)
fp.write(' ' * (h_len - fp.tell() - 1))
fp.close()