<commit_before>from enum import Enum

class BitmaskEnum(int, Enum):
    def __new__(cls, value=None):
        if value is None:
            if len(cls.__members__) == 0:
                value = 0
            else:
                value = 1 << (len(cls.__members__) - 1)
        
        obj = int.__new__(cls, value)
        obj._value_ = value
        return obj
    
    @classmethod
    def isPresent(cls,x,magic_num):
        return (x & magic_num) != 0
    
    @classmethod
    def strongest(cls,magic_num):
        if magic_num == 0:
            return cls(0)
        else:
            return cls(1 << (magic_num.bit_length() - 1))
    
    @classmethod
    def iterate(cls,magic_num):
        s = cls.strongest(magic_num)
        while s > 0:
            if (s & magic_num) != 0:
                yield cls(s)
            s >>= 1
    
    @classmethod
    def fromString(cls,s):
        try:
            return getattr(cls, s)
        except AttributeError:
            raise NotImplementedError("The {0} `{1}` is not implemented.".format(cls.__name__, s))

class Reduction(BitmaskEnum):
    none = 0
    w = 1 << 0
    RCA = 1 << 1
    c = 1 << 2
    sc = 1 << 3
    gW = 1 << 4
    W = 1 << 5
    sW = 1 << 6
    
    @classmethod<commit_msg>Optimized iteration over bitmasks<commit_after>from enum import Enum
from functools import lru_cache

class BitmaskEnum(int, Enum):
    def __new__(cls, value=None):
        if value is None:
            if len(cls.__members__) == 0:
                value = 0
            else:
                value = 1 << (len(cls.__members__) - 1)
        
        obj = int.__new__(cls, value)
        obj._value_ = value
        return obj
    
    @classmethod
    def isPresent(cls,x,magic_num):
        return (x & magic_num) != 0
    
    @classmethod
    @lru_cache(maxsize=32)
    def strongest(cls,magic_num):
        if magic_num == 0:
            return cls(0)
        else:
            return cls(1 << (magic_num.bit_length() - 1))
    
    @classmethod
    @lru_cache(maxsize=32)
    def iterate(cls,magic_num):
        return [x for x in cls if x & magic_num != 0]
    
    @classmethod
    def fromString(cls,s):
        try:
            return getattr(cls, s)
        except AttributeError:
            raise NotImplementedError("The {0} `{1}` is not implemented.".format(cls.__name__, s))

class Reduction(BitmaskEnum):
    none = 0
    w = 1 << 0
    RCA = 1 << 1
    c = 1 << 2
    sc = 1 << 3
    gW = 1 << 4
    W = 1 << 5
    sW = 1 << 6
    
    @classmethod
{'fraction_non_alphanumeric': 0.11199364575059571, 'fraction_numerical': 0.023828435266084195, 'mean_word_length': 1.6599788806758184, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1944788', 'n_tokens_mistral': 880, 'n_tokens_neox': 820, 'n_words': 225}