<commit_before>from clay.parserlib import *
from clay import tokens as t
from clay import lexer
from clay.ast import *
from clay.error import Location, raise_error

__all__ = ["parse"]

#
# primitives
#

def token_type(klass) :
    def predicate(x) : return type(x) is klass
    return modify(condition(predicate), lambda x : x.value)

def token_value(klass, value) :
    def predicate(x) :
        return (type(x) is klass) and (x.value == value)
    return modify(condition(predicate), lambda x : x.value)

def symbol(s) : return token_value(t.Symbol, s)
def keyword(s) : return token_value(t.Keyword, s)

identifier = token_type(t.Identifier)

semicolon = symbol(";")
comma = symbol(",")


#
# astnode
#

def astnode(parser, constructor) :
    def parser_proc(input) :
        start_tok = input.pos
        result = parser(input)
        if result is Failure :
            return Failure
        node = constructor(result)
        node.location = input.data[start_tok].location
        return node
    return parser_proc


#
# literals
#

string_literal = astnode(token_type(t.StringLiteral),
                         lambda x : StringLiteral(x))
char_literal = astnode(token_type(t.CharLiteral), lambda x : CharLiteral(x))
int_literal = astnode(token_type(t.IntLiteral), lambda x : IntLiteral(x))
bool_literal = choice(astnode(keyword("true"), lambda x : BoolLiteral(True)),
                      astnode(keyword("false"), lambda x : BoolLiteral(False)))
literal = choice(bool_literal, int_literal, char_literal, string_literal)


#
# atomic expr
#

expression2 = lambda input : expression(input)
expression_list = listof(expression2, comma)
opt_expression_list = modify(optional(expression_list),
                             lambda x : [] if x is None else x)

array_expr = astnode(sequence(symbol("["), expression_list, symbol("]")),
                     lambda x : ArrayExpr(x[1]))
tuple_expr = astnode(sequence(symbol("("), expression_list, symbol(")")),
                     lambda x : TupleExpr(x[1]))
name_ref = astnode(identifier, lambda x : NameRef(x))
atomic_expr = choice(array_expr, tuple_expr, name_ref, literal)


#
# suffix expr
#

name = astnode(identifier, lambda x : Name(x))

index_suffix = astnode(sequence(symbol("["), expression_list, symbol("]")),
                       lambda x : IndexExpr(None,x[1]))
call_suffix = astnode(sequence(symbol("("), opt_expression_list, symbol(")")),
                      lambda x : CallExpr(None,x[1]))
field_ref_suffix = astnode(sequence(symbol("."), name),
                           lambda x : FieldRef(None,x[1]))
tuple_ref_suffix = astnode(sequence(symbol("."), int_literal),
                           lambda x : TupleRef(None,x[1]))
pointer_ref_suffix = astnode(symbol("^"), lambda x : PointerRef(None))

suffix = choice(index_suffix, call_suffix, field_ref_suffix,
                tuple_ref_suffix, pointer_ref_suffix)

def fold_suffixes(expr, suffixes) :
    for suffix in suffixes :
        suffix.expr = expr
        suffix.location = expr.location
        expr = suffix
    return expr

suffix_expr = modify(sequence(atomic_expr, zeroplus(suffix)),
                     lambda x : fold_suffixes(x[0], x[1]))


#
# expression
#

address_of_expr = astnode(sequence(symbol("&"), suffix_expr),
                          lambda x : AddressOfExpr(x[1]))

expression = choice(address_of_expr, suffix_expr)


#
# type spec, variable, variable_list
#

type_spec = modify(sequence(symbol(":"), expression), lambda x : x[1])<commit_msg>fixed location setting for suffix expressions.<commit_after>from clay.parserlib import *
from clay import tokens as t
from clay import lexer
from clay.ast import *
from clay.error import Location, raise_error

__all__ = ["parse"]

#
# primitives
#

def token_type(klass) :
    def predicate(x) : return type(x) is klass
    return modify(condition(predicate), lambda x : x.value)

def token_value(klass, value) :
    def predicate(x) :
        return (type(x) is klass) and (x.value == value)
    return modify(condition(predicate), lambda x : x.value)

def symbol(s) : return token_value(t.Symbol, s)
def keyword(s) : return token_value(t.Keyword, s)

identifier = token_type(t.Identifier)

semicolon = symbol(";")
comma = symbol(",")


#
# astnode
#

def astnode(parser, constructor) :
    def parser_proc(input) :
        start_tok = input.pos
        result = parser(input)
        if result is Failure :
            return Failure
        node = constructor(result)
        node.location = input.data[start_tok].location
        return node
    return parser_proc


#
# literals
#

string_literal = astnode(token_type(t.StringLiteral),
                         lambda x : StringLiteral(x))
char_literal = astnode(token_type(t.CharLiteral), lambda x : CharLiteral(x))
int_literal = astnode(token_type(t.IntLiteral), lambda x : IntLiteral(x))
bool_literal = choice(astnode(keyword("true"), lambda x : BoolLiteral(True)),
                      astnode(keyword("false"), lambda x : BoolLiteral(False)))
literal = choice(bool_literal, int_literal, char_literal, string_literal)


#
# atomic expr
#

expression2 = lambda input : expression(input)
expression_list = listof(expression2, comma)
opt_expression_list = modify(optional(expression_list),
                             lambda x : [] if x is None else x)

array_expr = astnode(sequence(symbol("["), expression_list, symbol("]")),
                     lambda x : ArrayExpr(x[1]))
tuple_expr = astnode(sequence(symbol("("), expression_list, symbol(")")),
                     lambda x : TupleExpr(x[1]))
name_ref = astnode(identifier, lambda x : NameRef(x))
atomic_expr = choice(array_expr, tuple_expr, name_ref, literal)


#
# suffix expr
#

name = astnode(identifier, lambda x : Name(x))

index_suffix = astnode(sequence(symbol("["), expression_list, symbol("]")),
                       lambda x : IndexExpr(None,x[1]))
call_suffix = astnode(sequence(symbol("("), opt_expression_list, symbol(")")),
                      lambda x : CallExpr(None,x[1]))
field_ref_suffix = astnode(sequence(symbol("."), name),
                           lambda x : FieldRef(None,x[1]))
tuple_ref_suffix = astnode(sequence(symbol("."), int_literal),
                           lambda x : TupleRef(None,x[1]))
pointer_ref_suffix = astnode(symbol("^"), lambda x : PointerRef(None))

suffix = choice(index_suffix, call_suffix, field_ref_suffix,
                tuple_ref_suffix, pointer_ref_suffix)

def fold_suffixes(expr, suffixes) :
    for suffix in suffixes :
        suffix.expr = expr
        expr = suffix
    return expr

suffix_expr = modify(sequence(atomic_expr, zeroplus(suffix)),
                     lambda x : fold_suffixes(x[0], x[1]))


#
# expression
#

address_of_expr = astnode(sequence(symbol("&"), suffix_expr),
                          lambda x : AddressOfExpr(x[1]))

expression = choice(address_of_expr, suffix_expr)


#
# type spec, variable, variable_list
#

type_spec = modify(sequence(symbol(":"), expression), lambda x : x[1])
{'fraction_non_alphanumeric': 0.12473179802603347, 'fraction_numerical': 0.0034329852667715635, 'mean_word_length': 3.391959798994975, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4066948', 'n_tokens_mistral': 2171, 'n_tokens_neox': 2106, 'n_words': 508}