<commit_before>from rply.errors import ParserGeneratorError


def rightmost_terminal(symbols, terminals):
    for sym in reversed(symbols):
        if sym in terminals:
            return sym
    return None


class Grammar(object):
    def __init__(self, terminals):
        # A list of all the productions
        self.productions = [None]
        # A dictionary mapping the names of non-terminals to a list of all
        # productions of that nonterminal
        self.prod_names = {}
        # A dictionary mapping the names of terminals to a list of the rules
        # where they are used
        self.terminals = dict.fromkeys(terminals, [])
        # A dictionary mapping names of nonterminals to a list of rule numbers
        # where they are used
        self.nonterminals = {}
        self.first = {}
        self.follow = {}
        self.precedence = {}
        self.start = None

    def add_production(self, prod_name, syms, func, precedence):
        if prod_name in self.terminals:
            raise ParserGeneratorError("Illegal rule name %s" % prod_name)

        precname = precedence if precedence is not None else rightmost_terminal(syms, self.terminals)
        prod_prec = self.precedence.get(precname, ("right", 0))

        pnumber = len(self.productions)
        self.nonterminals.setdefault(prod_name, [])

        for t in syms:
            if t in self.terminals:
                self.terminals[t].append(pnumber)
            else:
                self.nonterminals.setdefault(t, []).append(pnumber)

        p = Production(pnumber, prod_name, syms, prod_prec, func)
        self.productions.append(p)

        self.prod_names.setdefault(prod_name, []).append(p)

    def set_precedence(self, term, assoc, level):
        if term in self.precedence:
            raise ParserGeneratorError("Precedence already specified for %s" % term)
        if assoc not in {"left", "right", "nonassoc"}:
            raise ParserGeneratorError("Precedence must be one of left, right, nonassoc; not %s" % assoc)
        self.precedence[term] = (assoc, level)

    def set_start(self):
        start = self.productions[1].name
        self.productions[0] = Production(0, "S'", [start], ("right", 0), None)
        self.nonterminals[start].append(0)
        self.start = start

    def build_lritems(self):
        """
        Walks the list of productions and builds a complete set of the LR
        items.
        """
        for p in self.productions:
            lastlri = p
            i = 0
            lr_items = []
            while True:
                if i > p.getlength():
                    lri = None
                else:
                    prod = p.prod[:]
                    prod.insert(i, ".")
                    try:
                        before = prod[i - 1]
                    except IndexError:
                        before = None
                    try:
                        after = self.prod_names[prod[i + 1]]
                    except (IndexError, KeyError):
                        after = []
                    lri = LRItem(p, i, before, after)
                lastlri.lr_next = lri
                if lri is None:
                    break
                lr_items.append(lri)
                lastlri = lri
                i += 1
            p.lr_items = lr_items

    def _first(self, beta):<commit_msg>simplify and make more efficient some code<commit_after>from rply.errors import ParserGeneratorError


def rightmost_terminal(symbols, terminals):
    for sym in reversed(symbols):
        if sym in terminals:
            return sym
    return None


class Grammar(object):
    def __init__(self, terminals):
        # A list of all the productions
        self.productions = [None]
        # A dictionary mapping the names of non-terminals to a list of all
        # productions of that nonterminal
        self.prod_names = {}
        # A dictionary mapping the names of terminals to a list of the rules
        # where they are used
        self.terminals = dict.fromkeys(terminals, [])
        # A dictionary mapping names of nonterminals to a list of rule numbers
        # where they are used
        self.nonterminals = {}
        self.first = {}
        self.follow = {}
        self.precedence = {}
        self.start = None

    def add_production(self, prod_name, syms, func, precedence):
        if prod_name in self.terminals:
            raise ParserGeneratorError("Illegal rule name %s" % prod_name)

        precname = precedence if precedence is not None else rightmost_terminal(syms, self.terminals)
        prod_prec = self.precedence.get(precname, ("right", 0))

        pnumber = len(self.productions)
        self.nonterminals.setdefault(prod_name, [])

        for t in syms:
            if t in self.terminals:
                self.terminals[t].append(pnumber)
            else:
                self.nonterminals.setdefault(t, []).append(pnumber)

        p = Production(pnumber, prod_name, syms, prod_prec, func)
        self.productions.append(p)

        self.prod_names.setdefault(prod_name, []).append(p)

    def set_precedence(self, term, assoc, level):
        if term in self.precedence:
            raise ParserGeneratorError("Precedence already specified for %s" % term)
        if assoc not in {"left", "right", "nonassoc"}:
            raise ParserGeneratorError("Precedence must be one of left, right, nonassoc; not %s" % assoc)
        self.precedence[term] = (assoc, level)

    def set_start(self):
        start = self.productions[1].name
        self.productions[0] = Production(0, "S'", [start], ("right", 0), None)
        self.nonterminals[start].append(0)
        self.start = start

    def build_lritems(self):
        """
        Walks the list of productions and builds a complete set of the LR
        items.
        """
        for p in self.productions:
            lastlri = p
            i = 0
            lr_items = []
            while True:
                if i > p.getlength():
                    lri = None
                else:
                    try:
                        before = p.prod[i - 1]
                    except IndexError:
                        before = None
                    try:
                        after = self.prod_names[p.prod[i]]
                    except (IndexError, KeyError):
                        after = []
                    lri = LRItem(p, i, before, after)
                lastlri.lr_next = lri
                if lri is None:
                    break
                lr_items.append(lri)
                lastlri = lri
                i += 1
            p.lr_items = lr_items

    def _first(self, beta):
{'fraction_non_alphanumeric': 0.08857913669064749, 'fraction_numerical': 0.0028477218225419665, 'mean_word_length': 1.678843837816138, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4113336', 'n_tokens_mistral': 1923, 'n_tokens_neox': 1798, 'n_words': 541}