<commit_before><commit_msg>Add a tool to convert hdf5 to bigfile.<commit_after>import h5py
import bigfile
import logging
import argparse

ap = argparse.ArgumentParser('h5tobigfile')
ap.add_argument("hdf5")
ap.add_argument("bigfile")
ap.add_argument("--verify", action='store_true', default=False)
ap.add_argument("--include", action="append")
ap.add_argument("--exclude", action="append")

def traverse(hobj, operation, prefix=""):
    if hasattr(hobj, 'keys'):
        for k in hobj.keys():
            path = prefix + '/' + k
            traverse(hobj[k], operation, path)

    if hasattr(hobj, 'dtype'):
        if hobj.dtype.names:
            for field in hobj.dtype.names:
                path = prefix + field + '/'
                traverse(hobj[field], operation, path)
        else:
            operation(hobj, prefix)

def main(ns):

    hin = h5py.File(ns.hdf5, mode='r')
    bout = bigfile.BigFile(ns.bigfile, create=True)

    def create(hobj, path):
        print("Copying %s as %s" % (str(hobj), path))
        if hobj.dtype.shape and len(hobj.dtype.shape) > 1:
            raise TypeError("bigfile cannot handle tensor dtype : %s" % str(hobj.dtype))

        bb = bout.create(path, hobj.dtype, size=hobj.size)
        bb.attrs['ndarray.shape'] = hobj.shape
        bb.write(0, hobj[:].ravel().copy())

    traverse(hin, create)

    bout.close()

    if ns.verify:
        bout = bigfile.BigFile(ns.bigfile, create=False)
        def verify(hobj, path):
            print("Verifying %s against %s" % (str(hobj), path))
            bb = bout[path]
            assert (hobj.dtype == bb.dtype)
            assert (hobj.shape == bb.attrs['ndarray.shape']).all()
            assert (hobj[:].flat == bb[:]).all()
        traverse(hin, verify)
        bout.close()

    hin.close()
if __name__ == "__main__" :
    ns = ap.parse_args()

    main(ns)
{'fraction_non_alphanumeric': 0.15455035002692516, 'fraction_numerical': 0.004308023694130318, 'mean_word_length': 2.5730769230769233, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2445563', 'n_tokens_mistral': 634, 'n_tokens_neox': 624, 'n_words': 138}