<commit_before>-module(ephp_parser).
-author('<EMAIL>').
-compile([warnings_as_errors, export_all]).

-export([parse/1, file/1]).

-include("ephp.hrl").
-include("ephp_parser.hrl").

-import(ephp_parser_expr, [
    expression/3, add_op/2, precedence/1
]).

file(File) ->
    {ok, Content} = file:read_file(File),
    parse(Content).

parse(Document) when is_list(Document) ->
    parse(list_to_binary(Document));
parse(Document) ->
    {_, _, Parsed} = document(Document, {root,1,1}, []),
    lists:reverse(Parsed).

document(<<>>, Pos, Parsed) ->
    {<<>>, Pos, Parsed};
document(<<"<?php",Rest/binary>>, {literal,_,_}=Pos, Parsed) ->
    {Rest, add_pos(Pos,5), Parsed};
document(<<"<?php",Rest/binary>>, Pos, Parsed) ->
    {Rest0,Pos0,NParsed} = code(Rest, normal_level(add_pos(Pos,5)), []),
    case NParsed of
        [] ->
            document(Rest0, Pos0, Parsed);
        _ ->
            Eval = add_line(#eval{statements=lists:reverse(NParsed)}, Pos),
            document(Rest0,Pos0,[Eval|Parsed])
    end;
document(<<"<?=",Rest/binary>>, Pos, Parsed) ->
    NewPos = code_value_level(add_pos(Pos,3)),
    {Rest0,Pos0,Text} = code(Rest, NewPos, []),
    document(Rest0, copy_level(Pos,Pos0), [get_print(Text,NewPos)|Parsed]);
document(<<"<?",Rest/binary>>, {literal,_,_}=Pos, Parsed) ->
    %% TODO: if short is not permitted, use as text
    {Rest, add_pos(Pos,2), Parsed};
document(<<"<?",Rest/binary>>, Pos, Parsed) ->
    %% TODO: if short is not permitted, use as text
    {Rest0,Pos0,NParsed} = code(Rest, normal_level(add_pos(Pos,2)), []),
    case NParsed of
        [] ->
            document(Rest0, Pos0, Parsed);
        _ ->
            Eval = add_line(#eval{statements=lists:reverse(NParsed)}, Pos),
            document(Rest0,Pos0,[Eval|Parsed])
    end;
document(<<"\n",Rest/binary>>, Pos, Parsed) ->
    document(Rest, new_line(Pos), add_to_text(<<"\n">>, Pos, Parsed));
document(<<L:1/binary,Rest/binary>>, Pos, Parsed) ->
    document(Rest, add_pos(Pos,1), add_to_text(L, Pos, Parsed)).

copy_level({Level,_,_}, {_,Row,Col}) -> {Level,Row,Col}.

code(<<>>, Pos, Parsed) ->
    {<<>>, Pos, Parsed};
code(<<B:8,R:8,E:8,A:8,K:8,SP:8,Rest/binary>>, Pos, Parsed) when
        ?OR(B,$B,$b) andalso ?OR(R,$R,$r) andalso ?OR(E,$E,$e) andalso
        ?OR(A,$A,$a) andalso ?OR(K,$K,$k) andalso
        (not (?IS_SPACE(SP) orelse ?IS_NUMBER(SP))) ->
    code(<<SP:8,Rest/binary>>, add_pos(Pos,5), [break|Parsed]);
code(<<C:8,O:8,N:8,T:8,I:8,N:8,U:8,E:8,SP:8,Rest/binary>>, Pos, Parsed) when
        ?OR(C,$C,$c) andalso ?OR(O,$O,$o) andalso ?OR(N,$N,$n) andalso
        ?OR(T,$T,$t) andalso ?OR(I,$I,$i) andalso ?OR(U,$U,$u) andalso
        ?OR(E,$E,$e) andalso (not (?IS_SPACE(SP) orelse ?IS_NUMBER(SP))) ->
    code(<<SP:8,Rest/binary>>, add_pos(Pos,8), [continue|Parsed]);
code(<<R:8,E:8,T:8,U:8,R:8,N:8,SP:8,Rest/binary>>, Pos, Parsed) when
        ?OR(R,$R,$r) andalso ?OR(E,$E,$e) andalso ?OR(T,$T,$t) andalso
        ?OR(U,$U,$u) andalso ?OR(N,$N,$n) andalso
        (not (?IS_ALPHA(SP) orelse ?IS_NUMBER(SP))) ->
    {Rest0, Pos0, Return} = expression(<<SP:8,Rest/binary>>, add_pos(Pos,6), []),
    case Return of
        [] -> code(Rest0, Pos0, [add_line(#return{}, Pos)|Parsed]);
        _ -> code(Rest0, Pos0, [add_line(#return{value = Return}, Pos)|Parsed])
    end;
code(<<T:8,H:8,R:8,O:8,W:8,SP:8,Rest/binary>>, Pos, Parsed) when
        ?OR(T,$T,$t) andalso ?OR(H,$H,$h) andalso ?OR(R,$R,$r) andalso
        ?OR(O,$O,$o) andalso ?OR(W,$W,$w) andalso
        (not (?IS_ALPHA(SP) orelse ?IS_NUMBER(SP) orelse SP =:= $_)) ->
    {Rest0, Pos0, Throw} = expression(<<SP:8,Rest/binary>>, add_pos(Pos,5), []),
    case Throw of
        [] -> throw_error(eparse, Pos0, Rest);
        _ -> code(Rest0, Pos0, [add_line(#throw{value = Throw}, Pos)|Parsed])
    end;
code(<<"@",Rest/binary>>, Pos, Parsed) ->
    {Rest0, Pos0, RParsed0} = code(Rest, add_pos(Pos,1), []),
    [ToSilent|Parsed0] = lists:reverse(RParsed0),<commit_msg>ensure a function with name 'break_...' is not handled as a break<commit_after>-module(ephp_parser).
-author('<EMAIL>').
-compile([warnings_as_errors, export_all]).

-export([parse/1, file/1]).

-include("ephp.hrl").
-include("ephp_parser.hrl").

-import(ephp_parser_expr, [
    expression/3, add_op/2, precedence/1
]).

file(File) ->
    {ok, Content} = file:read_file(File),
    parse(Content).

parse(Document) when is_list(Document) ->
    parse(list_to_binary(Document));
parse(Document) ->
    {_, _, Parsed} = document(Document, {root,1,1}, []),
    lists:reverse(Parsed).

document(<<>>, Pos, Parsed) ->
    {<<>>, Pos, Parsed};
document(<<"<?php",Rest/binary>>, {literal,_,_}=Pos, Parsed) ->
    {Rest, add_pos(Pos,5), Parsed};
document(<<"<?php",Rest/binary>>, Pos, Parsed) ->
    {Rest0,Pos0,NParsed} = code(Rest, normal_level(add_pos(Pos,5)), []),
    case NParsed of
        [] ->
            document(Rest0, Pos0, Parsed);
        _ ->
            Eval = add_line(#eval{statements=lists:reverse(NParsed)}, Pos),
            document(Rest0,Pos0,[Eval|Parsed])
    end;
document(<<"<?=",Rest/binary>>, Pos, Parsed) ->
    NewPos = code_value_level(add_pos(Pos,3)),
    {Rest0,Pos0,Text} = code(Rest, NewPos, []),
    document(Rest0, copy_level(Pos,Pos0), [get_print(Text,NewPos)|Parsed]);
document(<<"<?",Rest/binary>>, {literal,_,_}=Pos, Parsed) ->
    %% TODO: if short is not permitted, use as text
    {Rest, add_pos(Pos,2), Parsed};
document(<<"<?",Rest/binary>>, Pos, Parsed) ->
    %% TODO: if short is not permitted, use as text
    {Rest0,Pos0,NParsed} = code(Rest, normal_level(add_pos(Pos,2)), []),
    case NParsed of
        [] ->
            document(Rest0, Pos0, Parsed);
        _ ->
            Eval = add_line(#eval{statements=lists:reverse(NParsed)}, Pos),
            document(Rest0,Pos0,[Eval|Parsed])
    end;
document(<<"\n",Rest/binary>>, Pos, Parsed) ->
    document(Rest, new_line(Pos), add_to_text(<<"\n">>, Pos, Parsed));
document(<<L:1/binary,Rest/binary>>, Pos, Parsed) ->
    document(Rest, add_pos(Pos,1), add_to_text(L, Pos, Parsed)).

copy_level({Level,_,_}, {_,Row,Col}) -> {Level,Row,Col}.

code(<<>>, Pos, Parsed) ->
    {<<>>, Pos, Parsed};
code(<<B:8,R:8,E:8,A:8,K:8,SP:8,Rest/binary>>, Pos, Parsed) when
        ?OR(B,$B,$b) andalso ?OR(R,$R,$r) andalso ?OR(E,$E,$e) andalso
        ?OR(A,$A,$a) andalso ?OR(K,$K,$k) andalso
        (not (?IS_SPACE(SP) orelse ?IS_NUMBER(SP) orelse SP =:= $_)) ->
    code(<<SP:8,Rest/binary>>, add_pos(Pos,5), [break|Parsed]);
code(<<C:8,O:8,N:8,T:8,I:8,N:8,U:8,E:8,SP:8,Rest/binary>>, Pos, Parsed) when
        ?OR(C,$C,$c) andalso ?OR(O,$O,$o) andalso ?OR(N,$N,$n) andalso
        ?OR(T,$T,$t) andalso ?OR(I,$I,$i) andalso ?OR(U,$U,$u) andalso
        ?OR(E,$E,$e) andalso (not (?IS_SPACE(SP) orelse ?IS_NUMBER(SP))) ->
    code(<<SP:8,Rest/binary>>, add_pos(Pos,8), [continue|Parsed]);
code(<<R:8,E:8,T:8,U:8,R:8,N:8,SP:8,Rest/binary>>, Pos, Parsed) when
        ?OR(R,$R,$r) andalso ?OR(E,$E,$e) andalso ?OR(T,$T,$t) andalso
        ?OR(U,$U,$u) andalso ?OR(N,$N,$n) andalso
        (not (?IS_ALPHA(SP) orelse ?IS_NUMBER(SP))) ->
    {Rest0, Pos0, Return} = expression(<<SP:8,Rest/binary>>, add_pos(Pos,6), []),
    case Return of
        [] -> code(Rest0, Pos0, [add_line(#return{}, Pos)|Parsed]);
        _ -> code(Rest0, Pos0, [add_line(#return{value = Return}, Pos)|Parsed])
    end;
code(<<T:8,H:8,R:8,O:8,W:8,SP:8,Rest/binary>>, Pos, Parsed) when
        ?OR(T,$T,$t) andalso ?OR(H,$H,$h) andalso ?OR(R,$R,$r) andalso
        ?OR(O,$O,$o) andalso ?OR(W,$W,$w) andalso
        (not (?IS_ALPHA(SP) orelse ?IS_NUMBER(SP) orelse SP =:= $_)) ->
    {Rest0, Pos0, Throw} = expression(<<SP:8,Rest/binary>>, add_pos(Pos,5), []),
    case Throw of
        [] -> throw_error(eparse, Pos0, Rest);
        _ -> code(Rest0, Pos0, [add_line(#throw{value = Throw}, Pos)|Parsed])
    end;
code(<<"@",Rest/binary>>, Pos, Parsed) ->
    {Rest0, Pos0, RParsed0} = code(Rest, add_pos(Pos,1), []),
    [ToSilent|Parsed0] = lists:reverse(RParsed0),
{'fraction_non_alphanumeric': 0.23761755485893418, 'fraction_numerical': 0.020815047021943575, 'mean_word_length': 4.149128469980632, 'pattern_counts': {'":': 0, '<': 99, '<?xml version=': 0, '>': 141, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2839185', 'n_tokens_mistral': 3324, 'n_tokens_neox': 3300, 'n_words': 530}