<commit_before>from ..common import ParseError, UnexpectedToken, is_terminal
from .grammar_analysis import GrammarAnalyzer

class EndToken(str):
    type = '$end'

class Item:
    def __init__(self, rule, ptr, start, data):
        self.rule = rule
        self.ptr = ptr
        self.start = start
        self.data = data

    @property
    def expect(self):
        return self.rule.expansion[self.ptr]

    @property
    def is_complete(self):
        return self.ptr == len(self.rule.expansion)

    def advance(self, data):
        return Item(self.rule, self.ptr+1, self.start, self.data + [data])

    def __eq__(self, other):
        return self.start == other.start and self.ptr == other.ptr and self.rule == other.rule
    def __hash__(self):
        return hash((self.rule, self.ptr, self.start))

    def __repr__(self):
        before = map(str, self.rule.expansion[:self.ptr])
        after = map(str, self.rule.expansion[self.ptr:])
        return '<(%d) %s : %s * %s>' % (self.start, self.rule.origin, ' '.join(before), ' '.join(after))


class NewsList(list):
    def __init__(self, initial=None):
        list.__init__(self, initial or [])
        self.last_iter = 0

    def get_news(self):
        i = self.last_iter
        self.last_iter = len(self)
        return self[i:]


class Column:
    def __init__(self):
        self.to_reduce = NewsList()
        self.to_predict = NewsList()
        self.to_scan = NewsList()
        self.item_count = 0

    def add(self, items):
        self.item_count += len(items)
        for item in items:
            if item.is_complete:
                if item not in self.to_reduce:  # Avoid infinite loop
                    self.to_reduce.append(item)
            else:
                if is_terminal(item.expect):
                    self.to_scan.append(item)
                else:
                    self.to_predict.append(item)

    def __nonzero__(self):
        return bool(self.item_count)

class Parser:
    def __init__(self, parser_conf):<commit_msg>Tiny improvement in Earley<commit_after>from ..common import ParseError, UnexpectedToken, is_terminal
from .grammar_analysis import GrammarAnalyzer

class EndToken(str):
    type = '$end'

class Item:
    def __init__(self, rule, ptr, start, data):
        self.rule = rule
        self.ptr = ptr
        self.start = start
        self.data = data

    @property
    def expect(self):
        return self.rule.expansion[self.ptr]

    @property
    def is_complete(self):
        return self.ptr == len(self.rule.expansion)

    def advance(self, data):
        return Item(self.rule, self.ptr+1, self.start, self.data + [data])

    def __eq__(self, other):
        return self.start == other.start and self.ptr == other.ptr and self.rule == other.rule
    def __hash__(self):
        return hash((self.rule, self.ptr, self.start))

    def __repr__(self):
        before = map(str, self.rule.expansion[:self.ptr])
        after = map(str, self.rule.expansion[self.ptr:])
        return '<(%d) %s : %s * %s>' % (self.start, self.rule.origin, ' '.join(before), ' '.join(after))


class NewsList(list):
    def __init__(self, initial=None):
        list.__init__(self, initial or [])
        self.last_iter = 0

    def get_news(self):
        i = self.last_iter
        self.last_iter = len(self)
        return self[i:]


class Column:
    def __init__(self):
        self.to_reduce = NewsList()
        self.to_predict = NewsList()
        self.to_scan = NewsList()
        self.item_count = 0

    def add(self, items):
        self.item_count += len(items)
        for item in items:
            if item.is_complete:
                if item not in self.to_reduce:  # Avoid infinite loop
                    self.to_reduce.append(item)
            elif is_terminal(item.expect):
                self.to_scan.append(item)
            else:
                self.to_predict.append(item)

    def __nonzero__(self):
        return bool(self.item_count)

class Parser:
    def __init__(self, parser_conf):
{'fraction_non_alphanumeric': 0.11894710702756395, 'fraction_numerical': 0.0014899428855227217, 'mean_word_length': 2.2301523656776263, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6777396', 'n_tokens_mistral': 1275, 'n_tokens_neox': 1233, 'n_words': 301}