<commit_before>import base64
import re
import numpy as np

TIMEDELTA_DTYPE_RE = re.compile(r'\[(?P<dtype>[^\]]+)\]')


def encode_timedelta(td):
    dtype = TIMEDELTA_DTYPE_RE.search(str(td.dtype)).groupdict()['dtype']
    return base64.b64encode(td.astype(np.uint64).tostring()), dtype


def decode_timedelta(t):
    try:
        v = np.frombuffer(base64.b64decode(t[0]), dtype=np.uint64)[0]
        s = t[1]
        return np.timedelta64(long(v), s)
    except IndexError:
        return t
<commit_msg>Support the persistence of modules that accept arguments, provided that they have member variables that match the __init__ argument names<commit_after>import base64
import re
import numpy as np

TIMEDELTA_DTYPE_RE = re.compile(r'\[(?P<dtype>[^\]]+)\]')


def encode_timedelta(td):
    dtype = TIMEDELTA_DTYPE_RE.search(str(td.dtype)).groupdict()['dtype']
    return base64.b64encode(td.astype(np.uint64).tostring()), dtype


def decode_timedelta(t):
    try:
        v = np.frombuffer(base64.b64decode(t[0]), dtype=np.uint64)[0]
        s = t[1]
        return np.timedelta64(long(v), s)
    except IndexError:
        return t


def extract_init_args(instance):
    """
    Given an instance, and under the assumption that member variables have the
    same name as the __init__ arguments, extract the arguments so they can
    be used to reconstruct the instance when deserializing
    """
    cls = instance.__class__
    args = filter(
        lambda x: x != 'self',
        cls.__init__.im_func.func_code.co_varnames)
    return [instance.__dict__[key] for key in args]

{'fraction_non_alphanumeric': 0.12159594680177327, 'fraction_numerical': 0.024065864471184292, 'mean_word_length': 3.6745562130177514, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3071905', 'n_tokens_mistral': 552, 'n_tokens_neox': 534, 'n_words': 132}