<commit_before>from collections import defaultdict
import logging
import re

class FSA:
    def __init__(self, regex_transitions=True):
        self.states = set()
        self.input_alphabet = set()
        self.init_states = set()
        self.final_states = set()
        self.transitions = defaultdict(dict)
        self.active_states = None
        self.regex_transitions = regex_transitions

    def add_state(self, state, is_init=False, is_final=False):
        self.states.add(state)
        if is_init:
            self.set_init(state)
        if is_final:
            self.set_final(state)

    def add_states(self, states):
        for state in states:
            if isinstance(state, tuple) and len(state) == 3:
                self.add_state(*state)
            else:
                raise TypeError("states for FSA.add_states() has to be tuples")

    def set_init(self, state):
        if state not in self.states:
            raise ValueError("state to be init has to be in states already")
        else:
            self.init_states.add(state)

    def set_final(self, state):
        if state not in self.states:
            raise ValueError("state to be final has to be in states already")
        else:
            self.final_states.add(state)

    def add_transition(self, string, input_state, output_state):
        if input_state not in self.states or output_state not in self.states:
            raise ValueError("transition states has to be in states already")
        if self.regex_transitions:
            input_ = re.compile(string)
        else:
            input_ = string
            self.input_alphabet.add(string)
        self.transitions[input_state][input_] = output_state

    def check_states(self):
        if len(self.states) == 0:
            raise Exception("FSA has no states")
        if len(self.init_states) == 0:
            raise Exception("No init states in the FSA")
        if len(self.final_states) == 0:
            raise Exception("No final/acceptor states in the FSA")

    def init_active_states(self):
        self.active_states = set(self.init_states)

    def in_final(self):
        return len(self.active_states | self.final_states) > 0

    def read_symbol(self, string):
        self.check_states()
        if self.active_states is None:
            self.init_active_states()
        new_active_states = set() 
        for active_state in self.active_states:
            if self.regex_transitions:
                for trans_pattern, out_state in (
                        self.transitions[active_state].iteritems()):<commit_msg>modified:   fst.py - reset() added - bugfix in in_final()<commit_after>from collections import defaultdict
import logging
import re

class FSA:
    def __init__(self, regex_transitions=True):
        self.states = set()
        self.input_alphabet = set()
        self.init_states = set()
        self.final_states = set()
        self.transitions = defaultdict(dict)
        self.active_states = None
        self.regex_transitions = regex_transitions

    def add_state(self, state, is_init=False, is_final=False):
        self.states.add(state)
        if is_init:
            self.set_init(state)
        if is_final:
            self.set_final(state)

    def add_states(self, states):
        for state in states:
            if isinstance(state, tuple) and len(state) == 3:
                self.add_state(*state)
            else:
                raise TypeError("states for FSA.add_states() has to be tuples")

    def set_init(self, state):
        if state not in self.states:
            raise ValueError("state to be init has to be in states already")
        else:
            self.init_states.add(state)

    def set_final(self, state):
        if state not in self.states:
            raise ValueError("state to be final has to be in states already")
        else:
            self.final_states.add(state)

    def add_transition(self, string, input_state, output_state):
        if input_state not in self.states or output_state not in self.states:
            raise ValueError("transition states has to be in states already")
        if self.regex_transitions:
            input_ = re.compile(string)
        else:
            input_ = string
            self.input_alphabet.add(string)
        self.transitions[input_state][input_] = output_state

    def check_states(self):
        if len(self.states) == 0:
            raise Exception("FSA has no states")
        if len(self.init_states) == 0:
            raise Exception("No init states in the FSA")
        if len(self.final_states) == 0:
            raise Exception("No final/acceptor states in the FSA")

    def init_active_states(self):
        self.active_states = set(self.init_states)

    def reset(self):
        self.init_active_states()

    def in_final(self):
        return len(self.active_states & self.final_states) > 0

    def read_symbol(self, string):
        self.check_states()
        if self.active_states is None:
            self.init_active_states()
        new_active_states = set() 
        for active_state in self.active_states:
            if self.regex_transitions:
                for trans_pattern, out_state in (
                        self.transitions[active_state].iteritems()):
{'fraction_non_alphanumeric': 0.08557364634608103, 'fraction_numerical': 0.001893222264293828, 'mean_word_length': 2.199878861296184, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1393485', 'n_tokens_mistral': 1507, 'n_tokens_neox': 1473, 'n_words': 408}