<commit_before>import re
from inspect import isclass
from recognition.rules import tokens

TOKEN_PATTERNS = tuple((re.compile(p, re.I), _) for p, _ in (
    (r'[\n\t ]+', tokens.WhitespaceToken),
    (r'[a-z0-9]+', tokens.WordToken),
    (r'=.+', lambda text, stream: tokens.ActionSubstituteToken(text[1:], stream.defined_functions)),
    (r'<[a-z_]+[a-z0-9_]*>', lambda text, _: tokens.NamedRuleToken(text[1:-1])),
    (r'_((\d*-\d*)|\d+)', lambda text, stream: stream.read_repetition(text[1:])),
    (r'\|', tokens.OrToken),
    (r'\(', tokens.GroupingOpeningToken),
    (r'\)', tokens.GroupingClosingToken),
    (r'\[', tokens.OptionalGroupingOpeningToken),
    (r'\]', tokens.OptionalGroupingClosingToken),
))

class RuleTokenStream:

    def __init__(self, text, defined_functions=None):
        self.text = text
        self.defined_functions = {} if defined_functions is None else defined_functions

    def read_repetition(self, text):
        '''
        Ex: left_3 or (hello|goodbye)_3-9 for inclusive ranges
        Dash indicates range, low/high both optional, low defaults to
        0, high defaults to infinity
        '''
        assert text
        if '-' not in text:
            low, high = text, text
        else:
            low, high = text.split('-')
            low = low or 0
            high = high or None
        return tokens.RepetitionToken(low=low, high=high)

    def read_next_token(self, pos):
        for pattern, token_creator in TOKEN_PATTERNS:
            match = pattern.match(self.text, pos=pos) 
            if match:
                matched_text = self.text[pos:match.span()[-1]]
                token = token_creator(matched_text) if isclass(token_creator) else token_creator(matched_text, self)
                if hasattr(token, 'consumed_char_count'):
                    pos += token.consumed_char_count + 1
                else:
                    pos = match.span()[-1]
                return token, pos
        else:
            self.croak('uh oh')

    def __iter__(self):
        pos = 0
        while pos < len(self.text):
            token, pos = self.read_next_token(pos)
            yield token<commit_msg>Fix tokenize error message<commit_after>import re
from inspect import isclass
from recognition.rules import tokens

TOKEN_PATTERNS = tuple((re.compile(p, re.I), _) for p, _ in (
    (r'[\n\t ]+', tokens.WhitespaceToken),
    (r'[a-z0-9]+', tokens.WordToken),
    (r'=.+', lambda text, stream: tokens.ActionSubstituteToken(text[1:], stream.defined_functions)),
    (r'<[a-z_]+[a-z0-9_]*>', lambda text, _: tokens.NamedRuleToken(text[1:-1])),
    (r'_((\d*-\d*)|\d+)', lambda text, stream: stream.read_repetition(text[1:])),
    (r'\|', tokens.OrToken),
    (r'\(', tokens.GroupingOpeningToken),
    (r'\)', tokens.GroupingClosingToken),
    (r'\[', tokens.OptionalGroupingOpeningToken),
    (r'\]', tokens.OptionalGroupingClosingToken),
))

class RuleTokenStream:

    def __init__(self, text, defined_functions=None):
        self.text = text
        self.defined_functions = {} if defined_functions is None else defined_functions

    def read_repetition(self, text):
        '''
        Ex: left_3 or (hello|goodbye)_3-9 for inclusive ranges
        Dash indicates range, low/high both optional, low defaults to
        0, high defaults to infinity
        '''
        assert text
        if '-' not in text:
            low, high = text, text
        else:
            low, high = text.split('-')
            low = low or 0
            high = high or None
        return tokens.RepetitionToken(low=low, high=high)

    def read_next_token(self, pos):
        for pattern, token_creator in TOKEN_PATTERNS:
            match = pattern.match(self.text, pos=pos) 
            if match:
                matched_text = self.text[pos:match.span()[-1]]
                token = token_creator(matched_text) if isclass(token_creator) else token_creator(matched_text, self)
                if hasattr(token, 'consumed_char_count'):
                    pos += token.consumed_char_count + 1
                else:
                    pos = match.span()[-1]
                return token, pos
        self.croak(f'Cannot tokenize text: {self.text[pos:]}')

    def __iter__(self):
        pos = 0
        while pos < len(self.text):
            token, pos = self.read_next_token(pos)
            yield token
{'fraction_non_alphanumeric': 0.13269054178145087, 'fraction_numerical': 0.007805325987144169, 'mean_word_length': 2.393302180685358, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '812807', 'n_tokens_mistral': 1386, 'n_tokens_neox': 1339, 'n_words': 319}