<commit_before>## Based on PEP-354, copied and adapted from
## http://stackoverflow.com/questions/36932/

_enum_registry = {}


def enum_value_unpickler(names, value):
    enum_class = _enum_registry.get(names, None) or Enum(*names)
    return enum_class[value]

def Enum(*names):
    if names in _enum_registry:
        return _enum_registry[names]

    class EnumClass:
        __slots__ = names

        def __iter__(self):
            return iter(constants)

        def __len__(self):
            return len(constants)

        def __getitem__(self, i):
            return constants[i]

        def __repr__(self):
            return 'Enum' + str(names)

        def __str__(self):
            return 'enum ' + str(constants)

        def add_value(self, value):
            value = EnumValue(len(names))
            setattr(self, value, value)
            constants.append(value)

        def pull_up(self, cls):
            for name, value in zip(names, constants):
                setattr(cls, name, value)

    class EnumValue:
        __slots__ = ('__value', )
        Value = property(lambda self: self.__value)
        EnumType = property(lambda self: EnumType)

        def __init__(self, value):
            self.__value = value

        def __hash__(self):
            return hash(self.__value)

        def __lt__(self, other):
            return self.__value < other.__value

        def __eq__(self, other):
            return self.__value == other.__value

        def __nonzero__(self):
            return bool(self.__value)

        def __repr__(self):
            return str(names[self.__value])

        def __reduce__(self):
            return enum_value_unpickler, (names, self.__value)<commit_msg>enum.EnumValue is now derived from int<commit_after>## Based on PEP-354, copied and adapted from
## http://stackoverflow.com/questions/36932/

_enum_registry = {}


def enum_value_unpickler(names, value):
    enum_class = _enum_registry.get(names, None) or Enum(*names)
    return enum_class[value]

def Enum(*names):
    if names in _enum_registry:
        return _enum_registry[names]

    class EnumClass:
        __slots__ = names

        def __iter__(self):
            return iter(constants)

        def __len__(self):
            return len(constants)

        def __getitem__(self, i):
            return constants[i]

        def __repr__(self):
            return 'Enum' + str(names)

        def __str__(self):
            return 'enum ' + str(constants)

        def add_value(self, value):
            value = EnumValue(len(names))
            setattr(self, value, value)
            constants.append(value)

        def pull_up(self, cls):
            for name, value in zip(names, constants):
                setattr(cls, name, value)

    class EnumValue(int):
        EnumType = property(lambda self: EnumType)

        def __repr__(self):
            return str(names[self])

        __str__ = __repr__

        def __reduce__(self):
            return enum_value_unpickler, (names, int(self))
{'fraction_non_alphanumeric': 0.09749670619235837, 'fraction_numerical': 0.005270092226613966, 'mean_word_length': 2.1471502590673577, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2053037', 'n_tokens_mistral': 894, 'n_tokens_neox': 858, 'n_words': 221}