<commit_before>class AnyValue(object):
    def __repr__(self):
        return "<ANY VALUE>"

ANY_VALUE = AnyValue()

# XXX API isn't right yet; redefine in terms of the tuple instead of dict
class PredicateRegistry(object):
    def __init__(self, names):
        self.names = names
        self.key_to_value_id = {}
        self.values = {}
        self.counter = 0
        self.indexes = {}
        for name in names:
            self.indexes[name] = {}
            
    def register(self, key, value):        
        t = d_to_t(self.names, key)

        # make new value id
        value_id = self.counter
        self.values[value_id] = value
        self.counter += 1

        # get previous value id, if it is there
        old_value_id = self.key_to_value_id.get(t)
        if old_value_id is not None:
            del self.values[old_value_id]
    
        # record new value id
        self.key_to_value_id[t] = value_id
        
        for k, v in t:
            index = self.indexes[k]
            s = index.get(v)
            if s is None:
                index[v] = s = set()
            s.add(value_id)
            # old value id needs to be cleaned up
            if old_value_id is not None:
                s.discard(old_value_id)

    def _get_specific(self, t):
        result = None
        for k, v in t:
            index = self.indexes[k]
            s = index.get(v, set())
            if result is None:
                result = s
            else:
                result = result.intersection(s)
        assert 0 <= len(result) < 2 
        return result
            
    def get(self, key, default=None):
        t = d_to_t(self.names, key)
        for p in tuple_permutations(t):
            ids = self._get_specific(p)
            if ids:
                return self.values[ids.pop()]
        return default

def d_to_t(names, d):
    return tuple([(name, d.get(name, ANY_VALUE)) for name in names])

def tuple_permutations(t):
    first = t[0]
    rest = t[1:]
    k, v = first
    if not rest:
        yield (first,)
        if v is not ANY_VALUE:
            yield ((k, ANY_VALUE),)
        return
    ps = tuple_permutations(rest)
    for p in ps:
        yield (first,) + p
    ps = tuple_permutations(rest) # generator, so do this again
    if v is not ANY_VALUE:
        for p in ps:
            yield ((k, ANY_VALUE),) + p
<commit_msg>Can't I shortcut this code for a slight performance increase?<commit_after>class AnyValue(object):
    def __repr__(self):
        return "<ANY VALUE>"

ANY_VALUE = AnyValue()

# XXX API isn't right yet; redefine in terms of the tuple instead of dict
class PredicateRegistry(object):
    def __init__(self, names):
        self.names = names
        self.key_to_value_id = {}
        self.values = {}
        self.counter = 0
        self.indexes = {}
        for name in names:
            self.indexes[name] = {}
            
    def register(self, key, value):        
        t = d_to_t(self.names, key)

        # make new value id
        value_id = self.counter
        self.values[value_id] = value
        self.counter += 1

        # get previous value id, if it is there
        old_value_id = self.key_to_value_id.get(t)
        if old_value_id is not None:
            del self.values[old_value_id]
    
        # record new value id
        self.key_to_value_id[t] = value_id
        
        for k, v in t:
            index = self.indexes[k]
            s = index.get(v)
            if s is None:
                index[v] = s = set()
            s.add(value_id)
            # old value id needs to be cleaned up
            if old_value_id is not None:
                s.discard(old_value_id)

    def _get_specific(self, t):
        result = None
        for k, v in t:
            index = self.indexes[k]
            s = index.get(v, set())
            if result is None:
                result = s
            else:
                result = result.intersection(s)
            # XXX shortcut if result is empty?
        assert 0 <= len(result) < 2 
        return result
            
    def get(self, key, default=None):
        t = d_to_t(self.names, key)
        for p in tuple_permutations(t):
            ids = self._get_specific(p)
            if ids:
                return self.values[ids.pop()]
        return default

def d_to_t(names, d):
    return tuple([(name, d.get(name, ANY_VALUE)) for name in names])

def tuple_permutations(t):
    first = t[0]
    rest = t[1:]
    k, v = first
    if not rest:
        yield (first,)
        if v is not ANY_VALUE:
            yield ((k, ANY_VALUE),)
        return
    ps = tuple_permutations(rest)
    for p in ps:
        yield (first,) + p
    ps = tuple_permutations(rest) # generator, so do this again
    if v is not ANY_VALUE:
        for p in ps:
            yield ((k, ANY_VALUE),) + p

{'fraction_non_alphanumeric': 0.09378227638917579, 'fraction_numerical': 0.0024788266887006815, 'mean_word_length': 1.6018269747447609, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1266153', 'n_tokens_mistral': 1529, 'n_tokens_neox': 1493, 'n_words': 461}