<commit_before>import bisect

# This module keeps track of arbitrary "states" at any point of the code. 
# A state is considered known if every path to the given point agrees on
# its state, otherwise it is None (i.e. unknown). 

# It might be useful to be able to "freeze" the set of states by pushing 
# all state changes to the tips of the trees for fast reading. Perhaps this
# could be done on get_state, clearing the cache on set_state (assuming 
# incoming is immutable). 

# This module still needs a lot of work, and probably should totally be 
# redesigned. It doesn't take return, raise, continue, or break into 
# account. 

class ControlFlow:

    def __init__(self, start_pos, incoming, parent):
        self.start_pos = start_pos
        self.incoming = incoming
        if parent is None and incoming is not None:
            parent = incoming.parent
        self.parent = parent
        self.tip = {}
        self.end_pos = ((),)
        
    def start_branch(self, pos):
        self.end_pos = pos
        branch_point = BranchingControlFlow(pos, self)
        if self.parent is not None:
            self.parent.branches[-1] = branch_point
        return branch_point.branches[0]
    
    def next_branch(self, pos):
        self.end_pos = pos
        return self.parent.new_branch(pos)
        
    def finish_branch(self, pos):
        self.end_pos = pos
        self.parent.end_pos = pos
        return LinearControlFlow(pos, self.parent)
        
    def get_state(self, item, pos=((),())):
        return self.get_pos_state(item, pos)[1]
        
    def get_pos_state(self, item, pos=((),())):
        # do some caching
        if pos > self.end_pos:
            try:
                return self.tip[item]
            except KeyError:
                self.tip[item] = pos_state = self._get_pos_state(item, pos)
                return pos_state
        else:
            return self._get_pos_state(item, pos)
        
class LinearControlFlow(ControlFlow):<commit_msg>fixed end marker in control flow engine<commit_after>import bisect, sys

# This module keeps track of arbitrary "states" at any point of the code. 
# A state is considered known if every path to the given point agrees on
# its state, otherwise it is None (i.e. unknown). 

# It might be useful to be able to "freeze" the set of states by pushing 
# all state changes to the tips of the trees for fast reading. Perhaps this
# could be done on get_state, clearing the cache on set_state (assuming 
# incoming is immutable). 

# This module still needs a lot of work, and probably should totally be 
# redesigned. It doesn't take return, raise, continue, or break into 
# account. 

_END_POS = ((unichr(sys.maxunicode)*10),())

class ControlFlow:

    def __init__(self, start_pos, incoming, parent):
        self.start_pos = start_pos
        self.incoming = incoming
        if parent is None and incoming is not None:
            parent = incoming.parent
        self.parent = parent
        self.tip = {}
        self.end_pos = _END_POS
        
    def start_branch(self, pos):
        self.end_pos = pos
        branch_point = BranchingControlFlow(pos, self)
        if self.parent is not None:
            self.parent.branches[-1] = branch_point
        return branch_point.branches[0]
    
    def next_branch(self, pos):
        self.end_pos = pos
        return self.parent.new_branch(pos)
        
    def finish_branch(self, pos):
        self.end_pos = pos
        self.parent.end_pos = pos
        return LinearControlFlow(pos, self.parent)
        
    def get_state(self, item, pos=_END_POS):
        return self.get_pos_state(item, pos)[1]
        
    def get_pos_state(self, item, pos=_END_POS):
        # do some caching
        if pos > self.end_pos:
            try:
                return self.tip[item]
            except KeyError:
                self.tip[item] = pos_state = self._get_pos_state(item, pos)
                return pos_state
        else:
            return self._get_pos_state(item, pos)
        
class LinearControlFlow(ControlFlow):
{'fraction_non_alphanumeric': 0.07974403150381491, 'fraction_numerical': 0.0019689884321929607, 'mean_word_length': 2.322976287816844, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1856359', 'n_tokens_mistral': 1184, 'n_tokens_neox': 1159, 'n_words': 405}