<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import, unicode_literals
from collections import OrderedDict
from .grammars import *  # @UnusedWildImport
from .parsing import *  # @UnusedWildImport

__all__ = ['GrakoParser', 'GrakoGrammarGenerator']

class GrakoParserBase(Parser):

    def __init__(self, grammar_name, text, verbose=False):
        super(GrakoParserBase, self).__init__(text, ignorecase=True, verbose=verbose)
        self.grammar_name = grammar_name

    def parse(self, rule='grammar'):
        return super(GrakoParserBase, self).parse(rule)[rule]

    def _void_(self):
        self._token('()', 'void')

    def _token_(self):
        with self._choice_context():
            self._token("'")
            self._pattern(r"(?:[^'\\]|\\')*", 'token')
            self._token("'")
            return

        with self._choice_context():
            self._token('"')
            self._pattern(r'(?:[^"\\]|\\")*', 'token')
            self._token('"')
            return

        raise FailedParse(self._buffer, '<"> or' + "<'>")

    def _word_(self):
        self._pattern(r'[A-Za-z0-9_]+', 'word')

    def _call_(self):
        self._call('word', 'call')

    def _pattern_(self):
        self._token('?/')
        self._pattern(r'(.*?)(?=/\?)', 'pattern')
        self._token('/?')

    def _cut_(self):
        self._token('!', 'cut')

    def _eof_(self):
        self._token('<EOF>')

    def _subexp_(self):
        self._token('(')
        self._call('expre', 'exp')
        self._token(')')

    def _optional_(self):
        self._token('[')
        self._call('expre', 'optional')
        self._token(']')

    def _plus_(self):
        if not self._try('-', 'symbol'):
            self._token('+', 'symbol')

    def _repeat_(self):
        self._token('{')
        self._call('expre', 'repeat')
        self._token('}')
        try:
            self._call('plus', 'plus')
        except FailedParse:
            pass

    def _special_(self):
        self._token('?(')
        self._pattern(r'(.*)\)?', 'special')

    def _atom_(self):
        with self._choice_context():
            self._call('void', 'atom')
            return
        with self._choice_context():
            self._call('cut', 'atom')
            return
        with self._choice_context():
            self._call('token', 'atom')
            return
        with self._choice_context():
            self._call('call', 'atom')
            return
        with self._choice_context():
            self._call('pattern', 'atom')
            return
        with self._choice_context():
            self._call('eof', 'atom')
            return
        raise FailedParse(self._buffer, 'atom')


    def _term_(self):
        with self._choice_context():
            self._call('atom', 'term')
            return
        with self._choice_context():
            self._call('subexp', 'term')
            return
        with self._choice_context():
            self._call('repeat', 'term')
            return
        with self._choice_context():
            self._call('optional', 'term')
            return
        with self._choice_context():
            self._call('special', 'term')
            return
        raise FailedParse(self._buffer, 'term')

    def _named_(self):
        self._call('word', 'name')
        if not self._try('+:', 'force_list'):
            self._token(':')
        try:
            self._call('term', 'value')
            return
        except FailedParse as e:
            raise FailedCut(self._buffer, e)

    def _element_(self):
        with self._choice_context():
            self._call('named', 'element')
            return
        with self._choice_context():
            self._call('term', 'element')
            return
        raise FailedParse(self._buffer, 'element')

    def _sequence_(self):
        f = lambda : self._call('element', 'sequence', True)
        for _ in self._repeat_iterator(f):
            pass

    def _choice_(self):
        self._call('sequence', 'options', True)
        while True:
            p = self._pos<commit_msg>Disallow empty sequences in bootstrap parser.<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import, unicode_literals
from collections import OrderedDict
from .grammars import *  # @UnusedWildImport
from .parsing import *  # @UnusedWildImport

__all__ = ['GrakoParser', 'GrakoGrammarGenerator']

class GrakoParserBase(Parser):

    def __init__(self, grammar_name, text, verbose=False):
        super(GrakoParserBase, self).__init__(text, ignorecase=True, verbose=verbose)
        self.grammar_name = grammar_name

    def parse(self, rule='grammar'):
        return super(GrakoParserBase, self).parse(rule)[rule]

    def _void_(self):
        self._token('()', 'void')

    def _token_(self):
        with self._choice_context():
            self._token("'")
            self._pattern(r"(?:[^'\\]|\\')*", 'token')
            self._token("'")
            return

        with self._choice_context():
            self._token('"')
            self._pattern(r'(?:[^"\\]|\\")*', 'token')
            self._token('"')
            return

        raise FailedParse(self._buffer, '<"> or' + "<'>")

    def _word_(self):
        self._pattern(r'[A-Za-z0-9_]+', 'word')

    def _call_(self):
        self._call('word', 'call')

    def _pattern_(self):
        self._token('?/')
        self._pattern(r'(.*?)(?=/\?)', 'pattern')
        self._token('/?')

    def _cut_(self):
        self._token('!', 'cut')

    def _eof_(self):
        self._token('<EOF>')

    def _subexp_(self):
        self._token('(')
        self._call('expre', 'exp')
        self._token(')')

    def _optional_(self):
        self._token('[')
        self._call('expre', 'optional')
        self._token(']')

    def _plus_(self):
        if not self._try('-', 'symbol'):
            self._token('+', 'symbol')

    def _repeat_(self):
        self._token('{')
        self._call('expre', 'repeat')
        self._token('}')
        try:
            self._call('plus', 'plus')
        except FailedParse:
            pass

    def _special_(self):
        self._token('?(')
        self._pattern(r'(.*)\)?', 'special')

    def _atom_(self):
        with self._choice_context():
            self._call('void', 'atom')
            return
        with self._choice_context():
            self._call('cut', 'atom')
            return
        with self._choice_context():
            self._call('token', 'atom')
            return
        with self._choice_context():
            self._call('call', 'atom')
            return
        with self._choice_context():
            self._call('pattern', 'atom')
            return
        with self._choice_context():
            self._call('eof', 'atom')
            return
        raise FailedParse(self._buffer, 'atom')


    def _term_(self):
        with self._choice_context():
            self._call('atom', 'term')
            return
        with self._choice_context():
            self._call('subexp', 'term')
            return
        with self._choice_context():
            self._call('repeat', 'term')
            return
        with self._choice_context():
            self._call('optional', 'term')
            return
        with self._choice_context():
            self._call('special', 'term')
            return
        raise FailedParse(self._buffer, 'term')

    def _named_(self):
        self._call('word', 'name')
        if not self._try('+:', 'force_list'):
            self._token(':')
        try:
            self._call('term', 'value')
            return
        except FailedParse as e:
            raise FailedCut(self._buffer, e)

    def _element_(self):
        with self._choice_context():
            self._call('named', 'element')
            return
        with self._choice_context():
            self._call('term', 'element')
            return
        raise FailedParse(self._buffer, 'element')

    def _sequence_(self):
        self._call('element', 'sequence', True)
        f = lambda : self._call('element', 'sequence')
        for _ in self._repeat_iterator(f):
            pass

    def _choice_(self):
        self._call('sequence', 'options', True)
        while True:
            p = self._pos
{'fraction_non_alphanumeric': 0.1564748201438849, 'fraction_numerical': 0.0007194244604316547, 'mean_word_length': 1.9683274021352313, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3348231', 'n_tokens_mistral': 2453, 'n_tokens_neox': 2365, 'n_words': 496}