<commit_before>from collections import namedtuple, OrderedDict
from numbers import Number
import importlib
from pathlib import Path
import sys

import numpy as np
import h5py

from .callable import serialize_callable_and_test, deserialize_callable


isPython2 = sys.version_info.major == 2

if isPython2:
    strTypes = (str, unicode)
else:
    strTypes = (str, bytes)


def u(s):
    if isPython2:
        return unicode(s)
    else:
        return str(s)

# ------------------------------ IMPORTS ------------------------------ #

ListItem = namedtuple('ListItem', ['i', 'item'])


def load_list(node):
    unordered = []
    for i, (j, x) in enumerate(node.items()):
        unordered.append(ListItem(int(j), h5_import(x)))
    ordered = sorted(unordered)
    counts = [x.i for x in ordered]
    items = [x.item for x in ordered]
    if counts != list(range(len(counts))):
        raise ValueError("Attempted to import a list that is missing elements")
    return items


def load_tuple(node):
    return tuple(load_list(node))


def load_dict(node):
    imported_dict = {}
    for k, v in node.items():
        imported_dict[k] = h5_import(v)
    return imported_dict


def load_hdf5able(node):
    cls = import_hdf5able(node.attrs[attr_key_hdf5able_cls])
    serialized_d = load_dict(node)
    version = node.attrs[attr_key_hdf5able_version]
    d = cls.h5_dict_from_serialized_dict(serialized_d, version)
    return cls.h5_rebuild_from_dict(d)


def load_ndarray(node):
    return np.array(node)


def load_none(_):
    return None


def load_str(node):
    return u(np.array(node))


def load_bool(node):
    return bool(node.attrs[attr_key_bool_value])


def load_number(node):
    return np.asscalar(node.attrs[attr_key_number_value])


def load_path(node):
    return Path(load_str(node))


# ------------------------------ EXPORTS ------------------------------ #

zero_padded = lambda x: "{:0" + u(len(u(x))) + "}"


def save_list(parent, l, name):
    list_node = parent.create_group(name)
    padded = zero_padded(len(l))
    for i, x in enumerate(l):
        h5_export(list_node, x, padded.format(i))


def save_dict(parent, d, name):
    dict_node = parent.create_group(name)
    if sum(not isinstance(k, strTypes) for k in d.keys()) != 0:
        raise ValueError("Only dictionaries with string keys can be "
                         "serialized")
    for k, v in d.items():
        h5_export(dict_node, v, str(k))


def save_hdf5able(parent, hdf5able, name):
    # Objects behave a lot like dictionaries
    d = hdf5able.h5_dict_to_serializable_dict()
    save_dict(parent, d, name)
    # HDF5able added itself to the parent. Grab the node
    node = parent[name]
    # And set the attribute so it can be decoded.
    node.attrs[attr_key_hdf5able_cls] = str_of_cls(hdf5able.__class__)
    node.attrs[attr_key_hdf5able_version] = hdf5able.h5_version


def save_ndarray(parent, a, name):
    # fletcher32 is a checksum, lzf is fast OK compression
    parent.create_dataset(name, data=a, compression='gzip', fletcher32=True)


def save_none(parent, _, name):
    parent.create_group(name)  # A blank group


def save_str(parent, s, name):
    parent.create_dataset(name, data=s)


def save_bool(parent, a_bool, name):
    group = parent.create_group(name)  # A blank group
    group.attrs[attr_key_bool_value] = a_bool


def save_number(parent, a_number, name):<commit_msg>update comment<commit_after>from collections import namedtuple, OrderedDict
from numbers import Number
import importlib
from pathlib import Path
import sys

import numpy as np
import h5py

from .callable import serialize_callable_and_test, deserialize_callable


isPython2 = sys.version_info.major == 2

if isPython2:
    strTypes = (str, unicode)
else:
    strTypes = (str, bytes)


def u(s):
    if isPython2:
        return unicode(s)
    else:
        return str(s)

# ------------------------------ IMPORTS ------------------------------ #

ListItem = namedtuple('ListItem', ['i', 'item'])


def load_list(node):
    unordered = []
    for i, (j, x) in enumerate(node.items()):
        unordered.append(ListItem(int(j), h5_import(x)))
    ordered = sorted(unordered)
    counts = [x.i for x in ordered]
    items = [x.item for x in ordered]
    if counts != list(range(len(counts))):
        raise ValueError("Attempted to import a list that is missing elements")
    return items


def load_tuple(node):
    return tuple(load_list(node))


def load_dict(node):
    imported_dict = {}
    for k, v in node.items():
        imported_dict[k] = h5_import(v)
    return imported_dict


def load_hdf5able(node):
    cls = import_hdf5able(node.attrs[attr_key_hdf5able_cls])
    serialized_d = load_dict(node)
    version = node.attrs[attr_key_hdf5able_version]
    d = cls.h5_dict_from_serialized_dict(serialized_d, version)
    return cls.h5_rebuild_from_dict(d)


def load_ndarray(node):
    return np.array(node)


def load_none(_):
    return None


def load_str(node):
    return u(np.array(node))


def load_bool(node):
    return bool(node.attrs[attr_key_bool_value])


def load_number(node):
    return np.asscalar(node.attrs[attr_key_number_value])


def load_path(node):
    return Path(load_str(node))


# ------------------------------ EXPORTS ------------------------------ #

zero_padded = lambda x: "{:0" + u(len(u(x))) + "}"


def save_list(parent, l, name):
    list_node = parent.create_group(name)
    padded = zero_padded(len(l))
    for i, x in enumerate(l):
        h5_export(list_node, x, padded.format(i))


def save_dict(parent, d, name):
    dict_node = parent.create_group(name)
    if sum(not isinstance(k, strTypes) for k in d.keys()) != 0:
        raise ValueError("Only dictionaries with string keys can be "
                         "serialized")
    for k, v in d.items():
        h5_export(dict_node, v, str(k))


def save_hdf5able(parent, hdf5able, name):
    # Objects behave a lot like dictionaries
    d = hdf5able.h5_dict_to_serializable_dict()
    save_dict(parent, d, name)
    # HDF5able added itself to the parent. Grab the node
    node = parent[name]
    # And set the attribute so it can be decoded.
    node.attrs[attr_key_hdf5able_cls] = str_of_cls(hdf5able.__class__)
    node.attrs[attr_key_hdf5able_version] = hdf5able.h5_version


def save_ndarray(parent, a, name):
    # fletcher32 is a checksum, gzip compression is supported by Matlab
    parent.create_dataset(name, data=a, compression='gzip', fletcher32=True)


def save_none(parent, _, name):
    parent.create_group(name)  # A blank group


def save_str(parent, s, name):
    parent.create_dataset(name, data=s)


def save_bool(parent, a_bool, name):
    group = parent.create_group(name)  # A blank group
    group.attrs[attr_key_bool_value] = a_bool


def save_number(parent, a_number, name):
{'fraction_non_alphanumeric': 0.13932253313696613, 'fraction_numerical': 0.009131075110456554, 'mean_word_length': 3.189389265885256, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6369340', 'n_tokens_mistral': 2361, 'n_tokens_neox': 2298, 'n_words': 571}