<commit_before>from __future__ import print_function
from pprint import pprint
from rply import ParserGenerator, LexerGenerator
from rply.token import BaseBox

lg = LexerGenerator()
lg.add("LPAREN", r"\(")
lg.add("RPAREN", r"\)")
lg.add("ATOM", r"[^\s()]+")
lg.ignore(r"\s+")

pg = ParserGenerator(["LPAREN", "RPAREN", "ATOM"])


@pg.error
def error_handler(token):
    raise ValueError("unexpected %s" % token.gettokentype())


@pg.production("main : sexpr")
def main(p):
    return p[0]


@pg.production("sexpr : LPAREN expr RPAREN")
def sexpr(p):
    return BoxSexpr(p[1])


@pg.production("expr : ATOM")
def atom(p):
    return BoxAtom(p[0].getstr())


@pg.production("expr : ATOM sexpr")
@pg.production("expr : ATOM expr")
def sexpr(p):
    return BoxExpr(BoxAtom(p[0].getstr()), p[1])

lexer = lg.build()
parser = pg.build()


class BoxSexpr(BaseBox):
    def __init__(self, expr):
        self.expr = expr

    def expr(self):
        return self.expr

    def __str__(self):
        return "( %s )" % (self.expr)


class BoxExpr(BaseBox):
    def __init__(self, atom, expr):
        self.atom = atom
        self.expr = expr

    def atom(self):
        return self.atom

    def expr(self):
        return self.expr

    def __str__(self):
        return "%s . ( %s )" % (self.atom, self.expr)


class BoxAtom(BaseBox):
    def __init__(self, atom):
        self.atom = atom

    def atom(self):
        return self.atom

    def __str__(self):
        return "%s" % (self.atom)

print(parser.parse(lexer.lex("(+ 1 (* 3 2))")))<commit_msg>quote + simpler recursive descent (less boxing)<commit_after>from __future__ import print_function
from rply import ParserGenerator, LexerGenerator
from rply.token import BaseBox

lg = LexerGenerator()
lg.add("LPAREN", r"\(")
lg.add("RPAREN", r"\)")
lg.add("QUOTE", r"'")
lg.add("ATOM", r"[^\s()]+")
lg.ignore(r"\s+")

pg = ParserGenerator(["QUOTE", "LPAREN", "RPAREN", "ATOM"])


@pg.error
def error_handler(token):
    type = token.gettokentype()
    pos = token.getsourcepos()
    raise ValueError("unexpected %s at (%s, %s)" %
                     (type, pos.lineno, pos.colno))


@pg.production("main : sexpr")
def main(p):
    return p[0]


@pg.production("sexpr : ATOM")
@pg.production("sexpr : QUOTE sexpr")
@pg.production("sexpr : LPAREN list RPAREN")
def sexpr(p):
    if p[0].gettokentype() == "ATOM":
        return BoxAtom(p[0].getstr())
    elif p[0].gettokentype() == "QUOTE":
        return BoxQuote(p[1])
    else:
        return p[1]


@pg.production("list : sexpr")
@pg.production("list : sexpr list")
def list(p):
    if len(p) == 1:
        return BoxPair(p[0])
    else:
        return BoxPair(p[0], p[1])

lexer = lg.build()
parser = pg.build()


class BoxQuote(BaseBox):
    def __init__(self, sexpr):
        self.sexpr = sexpr

    def sexpr(self):
        return self.sexpr

    def __str__(self):
        return "' %s" % self.sexpr


class BoxPair(BaseBox):
    def __init__(self, x, y=None):
        self.x = x
        self.y = y

    def __str__(self):
        if self.y is None:
            y = "NIL"
        else:
            y = self.y
        return "(%s . %s)" % (self.x, y)


class BoxAtom(BaseBox):
    def __init__(self, atom):
        self.atom = atom

    def atom(self):
        return self.atom

    def __str__(self):
        return "%s" % (self.atom)


print(parser.parse(lexer.lex("1")))
print(parser.parse(lexer.lex("'1")))
print(parser.parse(lexer.lex("+")))
print(parser.parse(lexer.lex("'+")))
print(parser.parse(lexer.lex("(+ 1 2)")))
print(parser.parse(lexer.lex("'(+ 1 2)")))
print(parser.parse(lexer.lex("(+ 1 (* 3 2))")))
print(parser.parse(lexer.lex("'(+ 1 (* 3 2))")))
print(parser.parse(lexer.lex("(+ (* 4 5) (* 3 2))")))
print(parser.parse(lexer.lex("(+ '(* 4 5) (* 3 2))")))
{'fraction_non_alphanumeric': 0.18540068764877016, 'fraction_numerical': 0.010050251256281407, 'mean_word_length': 3.1651982378854626, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6284952', 'n_tokens_mistral': 1465, 'n_tokens_neox': 1390, 'n_words': 318}