<commit_before>%%% File        : aeso_parser.erl
%%% Author      : <NAME>
%%% Description :
%%% Created     : 1 Mar 2018 by <NAME>
-module(aeso_parser).

-export([string/1]).

-include("aeso_parse_lib.hrl").

-spec string(string()) -> {ok, [aeso_syntax:decl()]} | {error, {aeso_parse_lib:pos(), atom(), term()}}.
string(String) ->
  parse_and_scan(file(), String).

parse_and_scan(P, S) ->
  case aeso_scan:scan(S) of
    {ok, Tokens} -> aeso_parse_lib:parse(P, Tokens);
    Error        -> Error
  end.

%% -- Parsing rules ----------------------------------------------------------

file() -> choice([], block(decl())).

decl() ->
    ?LAZY_P(
    choice(
      %% Contract declaration
    [ ?RULE(keyword(contract), con(), tok('='), maybe_block(decl()), {contract, _1, _2, _4})

      %% Type declarations  TODO: format annotation for "type bla" vs "type bla()"
    , ?RULE(keyword(type), id(),                                   {type_decl, _1, _2, []})
    , ?RULE(keyword(type), id(), type_vars(),                      {type_decl, _1, _2, _3})
    , ?RULE(keyword(type), id(),              tok('='), typedef(), {type_def, _1, _2, [], _4})
    , ?RULE(keyword(type), id(), type_vars(), tok('='), typedef(), {type_def, _1, _2, _3, _5})

      %% Function declarations
    , ?RULE(modifiers(), keyword(function), id(), tok(':'), type(), {fun_decl, _2, _3, _5})     %% TODO modifiers
    , ?RULE(modifiers(), keyword(function), fundef(),               set_pos(get_pos(_2), _3))   %% TODO
    , ?RULE(keyword('let'),    valdef(),                            set_pos(get_pos(_1), _2))
    ])).

modifiers() ->
    many(choice([token(stateful), token(public), token(private), token(internal)])).

%% -- Type declarations ------------------------------------------------------

typedef() ->
    choice(
    [ ?RULE(type(),                   {alias_t, _1})
    , ?RULE(brace_list(field_type()), {record_t, _1})
    , ?RULE(constructors(),           {variant_t, _1})
    ]).

constructors() ->
    sep1(constructor(), tok('|')).

constructor() ->    %% TODO: format for Con() vs Con
    choice(?RULE(con(),              {constr_t, get_ann(_1), _1, []}),
           ?RULE(con(), type_args(), {constr_t, get_ann(_1), _1, _2})).

type_args()  -> paren_list(type()).
field_type() -> ?RULE(id(), tok(':'), type(), {field_t, get_ann(_1), immutable, _1, _3}).

%% -- Let declarations -------------------------------------------------------

letdecl() ->
    choice(
        ?RULE(keyword('let'), letdef(),                             set_pos(get_pos(_1), _2)),
        ?RULE(keyword('let'), tok(rec), sep1(letdef(), tok('and')), {letrec, _1, _3})).

letdef() -> choice(valdef(), fundef()).

valdef() ->
    choice(
        ?RULE(id(),                   tok('='), body(), {letval, [], _1, type_wildcard(), _3}),
        ?RULE(id(), tok(':'), type(), tok('='), body(), {letval, [], _1, _3, _5})).

fundef() ->
    choice(
    [ ?RULE(id(), args(),                   tok('='), body(), {letfun, [], _1, _2, type_wildcard(), _4})
    , ?RULE(id(), args(), tok(':'), type(), tok('='), body(), {letfun, [], _1, _2, _4, _6})
    ]).

args() -> paren_list(arg()).

arg() -> choice(
    ?RULE(id(),                   {arg, get_ann(_1), _1, type_wildcard()}),
    ?RULE(id(), tok(':'), type(), {arg, get_ann(_1), _1, _3})).

%% -- Types ------------------------------------------------------------------

type_vars() -> paren_list(tvar()).

type() -> ?LAZY_P(type100()).

type100() -> type200().

type200() ->
    ?RULE(many({fun_domain(), keyword('=>')}), type300(), fun_t(_1, _2)).

type300() -> type400().

type400() ->
    ?RULE(typeAtom(), optional(type_args()),
          case _2 of
            none       -> _1;
            {ok, Args} -> {app_t, get_ann(_1), _1, Args}
          end).

typeAtom() ->
    ?LAZY_P(choice(
    [ id(), token(con), token(qcon), token(qid), tvar()
    , ?RULE(keyword('('), comma_sep(type()), tok(')'), tuple_t(_1, _2))
    ])).

fun_domain() -> ?RULE(?LAZY_P(type300()), fun_domain(_1)).

%% -- Statements -------------------------------------------------------------

body() ->
    ?LET_P(Stmts, maybe_block(stmt()), block_e(Stmts)).

stmt() ->
    ?LAZY_P(choice(
    [ expr()
    , letdecl()
    , {switch, keyword(switch), parens(expr()), maybe_block(branch())}
    , {'if', keyword('if'), parens(expr()), body()}
    , {elif, keyword(elif), parens(expr()), body()}
    , {else, keyword(else), body()}
    ])).

branch() ->
    ?RULE(pattern(), keyword('=>'), body(), {'case', _2, _1, _3}).

pattern() ->
    ?LET_P(E, expr500(), parse_pattern(E)).

%% -- Expressions ------------------------------------------------------------

expr() -> expr100().

expr100() ->
    Expr100 = ?LAZY_P(expr100()),
    Expr200 = ?LAZY_P(expr200()),
    choice(
    [ ?RULE(args(), keyword('=>'), body(), {lam, _2, _1, _3})   %% TODO: better location
    , {'if', keyword('if'), parens(Expr100), Expr200, right(tok(else), Expr100)}
    , ?RULE(Expr200, optional(right(tok(':'), type())),
            case _2 of
                none       -> _1;
                {ok, Type} -> {typed, get_ann(_1), _1, Type}
            end)
    ]).

expr200() -> infixr(expr300(), binop('||')).
expr300() -> infixr(expr400(), binop('&&')).
expr400() -> infix(expr500(),  binop(['<', '>', '=<', '>=', '==', '!='])).
expr500() -> infixr(expr600(), binop(['::', '++'])).
expr600() -> infixl(expr650(), binop(['+', '-', 'bor', 'bxor', 'bsr', 'bsl'])).
expr650() -> ?RULE(many(token('-')), expr700(), prefixes(_1, _2)).
expr700() -> infixl(expr800(), binop(['*', '/', mod, 'band'])).
expr800() -> ?RULE(many(choice(token('!'), token('bnot'))), expr900(), prefixes(_1, _2)).
expr900() -> ?RULE(exprAtom(), many(elim()), elim(_1, _2)).

exprAtom() ->
    ?LAZY_P(begin
        Expr = ?LAZY_P(expr()),
        choice(
        [ id(), con(), token(qid), token(qcon)
        , token(hash), token(string), token(char)
        , token(int)
        , ?RULE(token(hex), set_ann(format, hex, setelement(1, _1, int)))
        , {bool, keyword(true), true}
        , {bool, keyword(false), false}
        , {record, [], brace_list(?LAZY_P(field_assignment()))}
        , {list, [], bracket_list(Expr)}
        , ?RULE(tok('['), Expr, binop('..'), Expr, tok(']'), _3(_2, _4))
        , ?RULE(keyword('('), comma_sep(Expr), tok(')'), tuple_e(_1, _2))
        ])
    end).

elim() ->
    ?LAZY_P(
    choice(
    [ {proj, keyword('.'), id()}
    , ?RULE(paren_list(expr()), {app, get_ann(_1), _1})
    , ?RULE(keyword('{'), comma_sep(field_assignment()), tok('}'), {rec_upd, _1, _2})
    , ?RULE(keyword('['), expr(), keyword(']'), {map_get, _1, _2})
    ])).

elim(E, [])                          -> E;
elim(E, [{proj, Ann, P} | Es])       -> elim({proj, Ann, E, P}, Es);
elim(E, [{app, Ann, Args} | Es])     -> elim({app, Ann, E, Args}, Es);
elim(E, [{rec_upd, Ann, Flds} | Es]) -> elim({record, Ann, E, Flds}, Es);
elim(E, [{map_get, Ann, Key} | Es])  -> elim({map_get, Ann, E, Key}, Es).


field_assignment() ->
    ?RULE(lvalue(), tok('='), expr(), {field, get_ann(_1), _1, _3}).


lvalue() ->
    ?LET_P(E, expr900(),
    case E of
        E = {proj, _, _, _}    -> E;
        E = {id, _, _}         -> E;
        E = {map_get, _, _, _} -> E;
        E -> bad_expr_err("Not a valid lvalue", E)
    end).
<commit_msg>Fix wrong annotation in parser<commit_after>%%% File        : aeso_parser.erl
%%% Author      : <NAME>
%%% Description :
%%% Created     : 1 Mar 2018 by <NAME>
-module(aeso_parser).

-export([string/1]).

-include("aeso_parse_lib.hrl").

-spec string(string()) -> {ok, [aeso_syntax:decl()]} | {error, {aeso_parse_lib:pos(), atom(), term()}}.
string(String) ->
  parse_and_scan(file(), String).

parse_and_scan(P, S) ->
  case aeso_scan:scan(S) of
    {ok, Tokens} -> aeso_parse_lib:parse(P, Tokens);
    Error        -> Error
  end.

%% -- Parsing rules ----------------------------------------------------------

file() -> choice([], block(decl())).

decl() ->
    ?LAZY_P(
    choice(
      %% Contract declaration
    [ ?RULE(keyword(contract), con(), tok('='), maybe_block(decl()), {contract, _1, _2, _4})

      %% Type declarations  TODO: format annotation for "type bla" vs "type bla()"
    , ?RULE(keyword(type), id(),                                   {type_decl, _1, _2, []})
    , ?RULE(keyword(type), id(), type_vars(),                      {type_decl, _1, _2, _3})
    , ?RULE(keyword(type), id(),              tok('='), typedef(), {type_def, _1, _2, [], _4})
    , ?RULE(keyword(type), id(), type_vars(), tok('='), typedef(), {type_def, _1, _2, _3, _5})

      %% Function declarations
    , ?RULE(modifiers(), keyword(function), id(), tok(':'), type(), {fun_decl, _2, _3, _5})     %% TODO modifiers
    , ?RULE(modifiers(), keyword(function), fundef(),               set_pos(get_pos(_2), _3))   %% TODO
    , ?RULE(keyword('let'),    valdef(),                            set_pos(get_pos(_1), _2))
    ])).

modifiers() ->
    many(choice([token(stateful), token(public), token(private), token(internal)])).

%% -- Type declarations ------------------------------------------------------

typedef() ->
    choice(
    [ ?RULE(type(),                   {alias_t, _1})
    , ?RULE(brace_list(field_type()), {record_t, _1})
    , ?RULE(constructors(),           {variant_t, _1})
    ]).

constructors() ->
    sep1(constructor(), tok('|')).

constructor() ->    %% TODO: format for Con() vs Con
    choice(?RULE(con(),              {constr_t, get_ann(_1), _1, []}),
           ?RULE(con(), type_args(), {constr_t, get_ann(_1), _1, _2})).

type_args()  -> paren_list(type()).
field_type() -> ?RULE(id(), tok(':'), type(), {field_t, get_ann(_1), immutable, _1, _3}).

%% -- Let declarations -------------------------------------------------------

letdecl() ->
    choice(
        ?RULE(keyword('let'), letdef(),                             set_pos(get_pos(_1), _2)),
        ?RULE(keyword('let'), tok(rec), sep1(letdef(), tok('and')), {letrec, _1, _3})).

letdef() -> choice(valdef(), fundef()).

valdef() ->
    choice(
        ?RULE(id(),                   tok('='), body(), {letval, [], _1, type_wildcard(), _3}),
        ?RULE(id(), tok(':'), type(), tok('='), body(), {letval, [], _1, _3, _5})).

fundef() ->
    choice(
    [ ?RULE(id(), args(),                   tok('='), body(), {letfun, [], _1, _2, type_wildcard(), _4})
    , ?RULE(id(), args(), tok(':'), type(), tok('='), body(), {letfun, [], _1, _2, _4, _6})
    ]).

args() -> paren_list(arg()).

arg() -> choice(
    ?RULE(id(),                   {arg, get_ann(_1), _1, type_wildcard()}),
    ?RULE(id(), tok(':'), type(), {arg, get_ann(_1), _1, _3})).

%% -- Types ------------------------------------------------------------------

type_vars() -> paren_list(tvar()).

type() -> ?LAZY_P(type100()).

type100() -> type200().

type200() ->
    ?RULE(many({fun_domain(), keyword('=>')}), type300(), fun_t(_1, _2)).

type300() -> type400().

type400() ->
    ?RULE(typeAtom(), optional(type_args()),
          case _2 of
            none       -> _1;
            {ok, Args} -> {app_t, get_ann(_1), _1, Args}
          end).

typeAtom() ->
    ?LAZY_P(choice(
    [ id(), token(con), token(qcon), token(qid), tvar()
    , ?RULE(keyword('('), comma_sep(type()), tok(')'), tuple_t(_1, _2))
    ])).

fun_domain() -> ?RULE(?LAZY_P(type300()), fun_domain(_1)).

%% -- Statements -------------------------------------------------------------

body() ->
    ?LET_P(Stmts, maybe_block(stmt()), block_e(Stmts)).

stmt() ->
    ?LAZY_P(choice(
    [ expr()
    , letdecl()
    , {switch, keyword(switch), parens(expr()), maybe_block(branch())}
    , {'if', keyword('if'), parens(expr()), body()}
    , {elif, keyword(elif), parens(expr()), body()}
    , {else, keyword(else), body()}
    ])).

branch() ->
    ?RULE(pattern(), keyword('=>'), body(), {'case', _2, _1, _3}).

pattern() ->
    ?LET_P(E, expr500(), parse_pattern(E)).

%% -- Expressions ------------------------------------------------------------

expr() -> expr100().

expr100() ->
    Expr100 = ?LAZY_P(expr100()),
    Expr200 = ?LAZY_P(expr200()),
    choice(
    [ ?RULE(args(), keyword('=>'), body(), {lam, _2, _1, _3})   %% TODO: better location
    , {'if', keyword('if'), parens(Expr100), Expr200, right(tok(else), Expr100)}
    , ?RULE(Expr200, optional(right(tok(':'), type())),
            case _2 of
                none       -> _1;
                {ok, Type} -> {typed, get_ann(_1), _1, Type}
            end)
    ]).

expr200() -> infixr(expr300(), binop('||')).
expr300() -> infixr(expr400(), binop('&&')).
expr400() -> infix(expr500(),  binop(['<', '>', '=<', '>=', '==', '!='])).
expr500() -> infixr(expr600(), binop(['::', '++'])).
expr600() -> infixl(expr650(), binop(['+', '-', 'bor', 'bxor', 'bsr', 'bsl'])).
expr650() -> ?RULE(many(token('-')), expr700(), prefixes(_1, _2)).
expr700() -> infixl(expr800(), binop(['*', '/', mod, 'band'])).
expr800() -> ?RULE(many(choice(token('!'), token('bnot'))), expr900(), prefixes(_1, _2)).
expr900() -> ?RULE(exprAtom(), many(elim()), elim(_1, _2)).

exprAtom() ->
    ?LAZY_P(begin
        Expr = ?LAZY_P(expr()),
        choice(
        [ id(), con(), token(qid), token(qcon)
        , token(hash), token(string), token(char)
        , token(int)
        , ?RULE(token(hex), set_ann(format, hex, setelement(1, _1, int)))
        , {bool, keyword(true), true}
        , {bool, keyword(false), false}
        , {record, [], brace_list(?LAZY_P(field_assignment()))}
        , {list, [], bracket_list(Expr)}
        , ?RULE(tok('['), Expr, binop('..'), Expr, tok(']'), _3(_2, _4))
        , ?RULE(keyword('('), comma_sep(Expr), tok(')'), tuple_e(_1, _2))
        ])
    end).

elim() ->
    ?LAZY_P(
    choice(
    [ {proj, keyword('.'), id()}
    , ?RULE(paren_list(expr()), {app, [], _1})
    , ?RULE(keyword('{'), comma_sep(field_assignment()), tok('}'), {rec_upd, _1, _2})
    , ?RULE(keyword('['), expr(), keyword(']'), {map_get, _1, _2})
    ])).

elim(E, [])                          -> E;
elim(E, [{proj, Ann, P} | Es])       -> elim({proj, Ann, E, P}, Es);
elim(E, [{app, Ann, Args} | Es])     -> elim({app, Ann, E, Args}, Es);
elim(E, [{rec_upd, Ann, Flds} | Es]) -> elim({record, Ann, E, Flds}, Es);
elim(E, [{map_get, Ann, Key} | Es])  -> elim({map_get, Ann, E, Key}, Es).


field_assignment() ->
    ?RULE(lvalue(), tok('='), expr(), {field, get_ann(_1), _1, _3}).


lvalue() ->
    ?LET_P(E, expr900(),
    case E of
        E = {proj, _, _, _}    -> E;
        E = {id, _, _}         -> E;
        E = {map_get, _, _, _} -> E;
        E -> bad_expr_err("Not a valid lvalue", E)
    end).

{'fraction_non_alphanumeric': 0.2923013585837793, 'fraction_numerical': 0.029710443255111844, 'mean_word_length': 2.3934807916181606, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 135, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2248946', 'n_tokens_mistral': 5519, 'n_tokens_neox': 5291, 'n_words': 1083}