<commit_before>from collections import OrderedDict

from torch import nn


class Module(nn.Module):
    def __init__(self):
        super().__init__()
        self.__dict__['_hyper'] = OrderedDict()

    def __getattr__(self, item):
        if '_hyper' in self.__dict__:
            _hyper = self.__dict__['_hyper']
            if item in _hyper:
                return _hyper[item]
        return nn.Module.__getattr__(self, item)

    def __setattr__(self, item, value):
        if '_hyper' in self.__dict__:
            _hyper = self.__dict__['_hyper']
            if item in _hyper:
                _hyper[item] = value
        return nn.Module.__setattr__(self, item, value)

    def hyper_parameters(self):
        return self.__dict__['_hyper']<commit_msg>rewrote dataset attributes to get rid of recarrays<commit_after>from collections import OrderedDict

from torch import nn


# class Module(nn.Module):
#     def __init__(self):
#         super().__init__()
#         self.__dict__['_hyper'] = OrderedDict()
#
#     def __getattr__(self, item):
#         if '_hyper' in self.__dict__:
#             _hyper = self.__dict__['_hyper']
#             if item in _hyper:
#                 return _hyper[item]
#         return nn.Module.__getattr__(self, item)
#
#     def __setattr__(self, item, value):
#         if '_hyper' in self.__dict__:
#             _hyper = self.__dict__['_hyper']
#             if item in _hyper:
#                 _hyper[item] = value
#         return nn.Module.__setattr__(self, item, value)
#
#     def hyper_parameters(self):
#         return self.__dict__['_hyper']

class ModuleDict(nn.Module):
    def __init__(self, modules=None):
        super().__init__()
        if modules is not None:
            for k, v in modules.items():
                if k not in self._modules:
                    self.add_module(str(k), v)

    def __getitem__(self, idx):
        return self._modules[str(idx)]

    def __setitem__(self, idx, module):
        return setattr(self, str(idx), module)

    def __len__(self):
        return len(self._modules)

    def __iter__(self):
        return iter(self._modules.values())


{'fraction_non_alphanumeric': 0.11167985109353187, 'fraction_numerical': 0.0, 'mean_word_length': 2.2477341389728096, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1629681', 'n_tokens_mistral': 642, 'n_tokens_neox': 626, 'n_words': 164}