<commit_before>%% Copyright (c) 2008,2009 <NAME>. All rights reserved.
%%
%% Redistribution and use in source and binary forms, with or without
%% modification, are permitted provided that the following conditions
%% are met:
%%
%% 1. Redistributions of source code must retain the above copyright
%% notice, this list of conditions and the following disclaimer.
%% 2. Redistributions in binary form must reproduce the above copyright
%% notice, this list of conditions and the following disclaimer in the
%% documentation and/or other materials provided with the distribution.
%%
%% THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
%% "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
%% LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
%% FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
%% COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
%% INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
%% BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
%% LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
%% CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
%% LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
%% ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
%% POSSIBILITY OF SUCH DAMAGE.

%%% A Lexical Analyser Generator for Erlang.
%%%
%%% Most of the algorithms used here are taken pretty much as
%%% described in the "Dragon Book" by Aho, Sethi and Ullman. Some
%%% completing details were taken from "Compiler Design in C" by
%%% Hollub.

-module(leex).

-export([compile/3,file/1,file/2,format_error/1]).

-import(lists, [member/2,reverse/1,sort/1,delete/2,
                keysort/2,keydelete/3,
                map/2,foldl/3,foreach/2,flatmap/2]).
-import(string, [substr/2,substr/3,span/2]).
-import(ordsets, [is_element/2,add_element/2,union/2]).
-import(orddict, [store/3]).

-include("erl_compile.hrl").
%%-include_lib("stdlib/include/erl_compile.hrl").

-define(LEEXINC, "leexinc.hrl").        % Include file
-define(LEEXLIB, parsetools).           % Leex is in lib parsetools
%%-define(LEEXLIB, leex).               % Leex is in lib leex

-define(DEFS_HEAD, "Definitions.").
-define(RULE_HEAD, "Rules.").
-define(CODE_HEAD, "Erlang code.").

-record(leex, {xfile=[],        % Xrl file
               efile=[],        % Erl file
               ifile=[],        % Include file
               gfile=[],        % Graph file
               module,          % Module name
               opts=[],         % Options
               encoding=none,   % Encoding of Xrl file
               % posix=false,   % POSIX regular expressions
               errors=[],
               warnings=[]
              }).

-record(nfa_state, {no,edges=[],accept=noaccept}).
-record(dfa_state, {no,nfa=[],trans=[],accept=noaccept}).

%%%
%%% Exported functions
%%%

%%% Interface to erl_compile.

compile(Input0, Output0,
        #options{warning = WarnLevel, verbose=Verbose, includes=Includes,
		 specific=Specific}) ->
    Input = assure_extension(shorten_filename(Input0), ".xrl"),
    Output = assure_extension(shorten_filename(Output0), ".erl"),
    Includefile = lists:sublist(Includes, 1),
    Werror = proplists:get_bool(warnings_as_errors, Specific),
    Opts = [{scannerfile,Output},{includefile,Includefile},{verbose,Verbose},
            {report_errors,true},{report_warnings,WarnLevel > 0},
	    {warnings_as_errors, Werror}],
    case file(Input, Opts) of
        {ok, _} ->
            ok;
        error ->
            error
    end.

%% file(File) -> ok | error.
%% file(File, Options) -> ok | error.

file(File) -> file(File, []).

file(File, Opts0) ->
    case is_filename(File) of
        no -> erlang:error(badarg, [File,Opts0]);
        _ -> ok
    end,
    Opts = case options(Opts0) of
               badarg ->
                   erlang:error(badarg, [File,Opts0]);
               Options ->
                   Options
           end,
    St0 = #leex{},
    St1 = filenames(File, Opts, St0),   % Get all the filenames
    St = try
             {ok,REAs,Actions,Code,St2} = parse_file(St1),
             {DFA,DF} = make_dfa(REAs, St2),
             case werror(St2) of
                 false ->
                     St3 = out_file(St2, DFA, DF, Actions, Code),
                     case lists:member(dfa_graph, St3#leex.opts) of
                         true -> out_dfa_graph(St3, DFA, DF);
                         false -> St3
                     end;
                 true ->
                     St2
             end
         catch #leex{}=St4 ->
             St4
         end,
    leex_ret(St).             

format_error({file_error, Reason}) ->
    io_lib:fwrite("~ts",[file:format_error(Reason)]);
format_error(missing_defs) -> "missing Definitions";
format_error(missing_rules) -> "missing Rules";
format_error(missing_code) -> "missing Erlang code";
format_error(empty_rules) -> "no rules";
format_error(bad_rule) -> "bad rule";
format_error({regexp,E})->
    Es = case E of
             {interval_range,_} -> "interval range";
             {unterminated,Cs} ->
                 "unterminated " ++ Cs;
             {illegal_char,Cs} ->
                 "illegal character " ++ Cs;
%%           {posix_cc,What} ->
%%               ["illegal POSIX character class ",io_lib:write_string(What)];
             {char_class,What} ->
                 ["illegal character class ",io_lib:write_string(What)]
         end,
    ["bad regexp `",Es,"'"];
format_error(ignored_characters) ->
    "ignored characters";
format_error(cannot_parse) ->
    io_lib:fwrite("cannot parse; probably encoding mismatch", []).

%%%
%%% Local functions
%%%

assure_extension(File, Ext) ->
    lists:concat([strip_extension(File, Ext), Ext]).

%% Assumes File is a filename.
strip_extension(File, Ext) ->
    case filename:extension(File) of
        Ext -> filename:rootname(File);
        _Other -> File
    end.

options(Options0) when is_list(Options0) ->
    try 
        Options = flatmap(fun(return) -> short_option(return, true);
                             (report) -> short_option(report, true);
                             ({return,T}) -> short_option(return, T);
                             ({report,T}) -> short_option(report, T);
                             (T) -> [T]
                          end, Options0),
        options(Options, [scannerfile,includefile,report_errors,
                          report_warnings,warnings_as_errors,
                          return_errors,return_warnings,
                          verbose,dfa_graph], [])
    catch error: _ -> badarg
    end;
options(Option) ->
    options([Option]).

short_option(return, T) ->
    [{return_errors,T}, {return_warnings,T}];
short_option(report, T) ->
    [{report_errors,T}, {report_warnings,T}].

options(Options0, [Key|Keys], L) when is_list(Options0) ->
    Options = case member(Key, Options0) of
                  true -> 
                      [atom_option(Key)|delete(Key, Options0)];
                  false ->
                      Options0
              end,
    V = case lists:keyfind(Key, 1, Options) of
            {Key, Filename0} when Key =:= includefile;
				  Key =:= scannerfile ->
                case is_filename(Filename0) of
                    no -> 
                        badarg;
                    Filename -> 
                        {ok,[{Key,Filename}]}
                end;
            {Key, Bool} = KB when is_boolean(Bool) ->
                {ok, [KB]};
            {Key, _} ->
                badarg;
            false ->
                {ok,[{Key,default_option(Key)}]}
        end,
    case V of
        badarg ->
            badarg;
        {ok,KeyValueL} ->
            NewOptions = keydelete(Key, 1, Options),
            options(NewOptions, Keys, KeyValueL ++ L)
    end;
options([], [], L) ->
    foldl(fun({_,false}, A) -> A;
             ({Tag,true}, A) -> [Tag|A];
             (F,A) -> [F|A]
          end, [], L);
options(_Options, _, _L) ->
    badarg.

default_option(dfa_graph) -> false;
default_option(includefile) -> [];
default_option(report_errors) -> true;
default_option(report_warnings) -> true;
default_option(warnings_as_errors) -> false;
default_option(return_errors) -> false;
default_option(return_warnings) -> false;
default_option(scannerfile) -> [];
default_option(verbose) -> false.

atom_option(dfa_graph) -> {dfa_graph,true};
atom_option(report_errors) -> {report_errors,true};
atom_option(report_warnings) -> {report_warnings,true};
atom_option(warnings_as_errors) -> {warnings_as_errors,true};
atom_option(return_errors) -> {return_errors,true};
atom_option(return_warnings) -> {return_warnings,true};
atom_option(verbose) -> {verbose,true};
atom_option(Key) -> Key.

is_filename(T) ->
    try filename:flatten(T)
    catch error: _ -> no
    end.    

shorten_filename(Name0) ->
    {ok,Cwd} = file:get_cwd(),
    case lists:prefix(Cwd, Name0) of
        false -> Name0;
        true ->
            case lists:nthtail(length(Cwd), Name0) of
                "/"++N -> N;
                N -> N
            end
    end.

leex_ret(St) ->
    report_errors(St),
    report_warnings(St),
    Es = pack_errors(St#leex.errors),
    Ws = pack_warnings(St#leex.warnings),
    Werror = werror(St),
    if 
        Werror ->
            do_error_return(St, Es, Ws);
        Es =:= [] -> 
            case member(return_warnings, St#leex.opts) of
                true -> {ok, St#leex.efile, Ws};
                false -> {ok, St#leex.efile}
            end;
        true ->
            do_error_return(St, Es, Ws)
    end.

do_error_return(St, Es, Ws) ->
    case member(return_errors, St#leex.opts) of
        true -> {error, Es, Ws};
        false -> error
    end.

werror(St) ->
    St#leex.warnings =/= []
	andalso member(warnings_as_errors, St#leex.opts).

pack_errors([{File,_} | _] = Es) ->
    [{File, flatmap(fun({_,E}) -> [E] end, sort(Es))}];
pack_errors([]) ->
    [].
    
pack_warnings([{File,_} | _] = Ws) ->
    [{File, flatmap(fun({_,W}) -> [W] end, sort(Ws))}];
pack_warnings([]) ->
    [].

report_errors(St) ->
    when_opt(fun () -> 
                     foreach(fun({File,{none,Mod,E}}) -> 
                                     io:fwrite("~ts: ~ts\n",
                                               [File,Mod:format_error(E)]);
                                ({File,{Line,Mod,E}}) -> 
                                     io:fwrite("~ts:~w: ~ts\n",
                                               [File,Line,Mod:format_error(E)])
                             end, sort(St#leex.errors))
             end, report_errors, St#leex.opts).

report_warnings(St) ->
    Werror = member(warnings_as_errors, St#leex.opts),
    Prefix = case Werror of
		 true -> "";
		 false -> "Warning: "
	     end,
    ReportWerror = Werror andalso member(report_errors, St#leex.opts),
    ShouldReport = member(report_warnings, St#leex.opts) orelse ReportWerror,
    when_bool(fun () ->
		      foreach(fun({File,{none,Mod,W}}) ->
				      io:fwrite("~ts: ~s~ts\n",
						[File,Prefix,
						 Mod:format_error(W)]);
				 ({File,{Line,Mod,W}}) ->
				      io:fwrite("~ts:~w: ~s~ts\n",
						[File,Line,Prefix,
						 Mod:format_error(W)])
			      end, sort(St#leex.warnings))
	      end, ShouldReport).

-spec add_error(_, #leex{}) -> no_return().
add_error(E, St) ->
    add_error(St#leex.xfile, E, St).

add_error(File, Error, St) ->
    throw(St#leex{errors = [{File,Error}|St#leex.errors]}).

add_warning(Line, W, St) ->
    St#leex{warnings = [{St#leex.xfile,{Line,leex,W}}|St#leex.warnings]}.

%% filenames(File, Options, State) -> State.
%%  The default output dir is the current directory unless an
%%  explicit one has been given in the options.

filenames(File, Opts, St0) ->
    Dir = filename:dirname(File),
    Base = filename:basename(File, ".xrl"),
    Xfile = filename:join(Dir, Base ++ ".xrl"),
    Efile = Base ++ ".erl",
    Gfile = Base ++ ".dot",
    Module = list_to_atom(Base),
    St1 = St0#leex{xfile=Xfile,
                   opts=Opts,
                   module=Module},
    {includefile,Ifile0} = lists:keyfind(includefile, 1, Opts),
    Ifile = inc_file_name(Ifile0),
    %% Test for explicit scanner file.
    {scannerfile,Ofile} = lists:keyfind(scannerfile, 1, Opts),
    if
        Ofile =:= [] ->
            St1#leex{efile=filename:join(Dir, Efile),
                     ifile=Ifile,
                     gfile=filename:join(Dir, Gfile)};
        true ->
            D = filename:dirname(Ofile),
            St1#leex{efile=Ofile,
                     ifile=Ifile,
                     gfile=filename:join(D, Gfile)}
    end.

when_opt(Do, Opt, Opts) ->
    case member(Opt, Opts) of
        true -> Do();
        false -> ok
    end.

when_bool(Do, Bool) ->
    case Bool of
	true -> Do();
	false -> ok
    end.

verbose_print(St, Format, Args) ->
    when_opt(fun () -> io:fwrite(Format, Args) end, verbose, St#leex.opts).

%% parse_file(State) -> {ok,[REA],[Action],Code,NewState} | throw(NewState)
%%  when
%%      REA = {RegExp,ActionNo};
%%      Action = {ActionNo,ActionString};
%%      Code = {StartLine, StartPos, NumOfLines}. Where the Erlang code is.
%%
%%  Read and parse the file Xfile.
%%  After each section of the file has been parsed we directly call the
%%  next section. This is done when we detect a line we don't recognise
%%  in the current section. The file format is very simple and Erlang
%%  token based, we allow empty lines and Erlang style comments.

parse_file(St0) ->
    case file:open(St0#leex.xfile, [read]) of
        {ok,Xfile} ->
            St1 = St0#leex{encoding = epp:set_encoding(Xfile)},
            try
                verbose_print(St1, "Parsing file ~ts, ", [St1#leex.xfile]),
                %% We KNOW that errors throw so we can ignore them here.
                {ok,Line1,St2} = parse_head(Xfile, St1),
                {ok,Line2,Macs,St3} = parse_defs(Xfile, Line1, St2),
                {ok,Line3,REAs,Actions,St4} =
                    parse_rules(Xfile, Line2, Macs, St3),
                {ok,Code,St5} = parse_code(Xfile, Line3, St4),
                verbose_print(St5, "contained ~w rules.~n", [length(REAs)]),
                {ok,REAs,Actions,Code,St5}
            after ok = file:close(Xfile)
            end;
        {error,Error} ->
            add_error({none,leex,{file_error,Error}}, St0)
    end.

%% parse_head(File, State) -> {ok,NextLine,State}.
%%  Parse the head of the file. Skip all comments and blank lines.

parse_head(Ifile, St) -> {ok,nextline(Ifile, 0, St),St}.

%% parse_defs(File, Line, State) -> {ok,NextLine,Macros,State}.
%%  Parse the macro definition section of a file. This must exist.
%%  The section is ended by a non-blank line which is not a macro def.

parse_defs(Ifile, {ok,?DEFS_HEAD ++ Rest,L}, St) ->
    St1 = warn_ignored_chars(L, Rest, St),
    parse_defs(Ifile, nextline(Ifile, L, St), [], St1);
parse_defs(_, {ok,_,L}, St) ->
    add_error({L,leex,missing_defs}, St);
parse_defs(_, {eof,L}, St) ->
    add_error({L,leex,missing_defs}, St).

parse_defs(Ifile, {ok,Chars,L}=Line, Ms, St) ->
    %% This little beauty matches out a macro definition, RE's are so clear.
    MS = "^[ \t]*([A-Z_][A-Za-z0-9_]*)[ \t]*=[ \t]*([^ \t\r\n]*)[ \t\r\n]*\$",
    case re:run(Chars, MS, [{capture,all_but_first,list},unicode]) of
        {match,[Name,Def]} ->
            %%io:fwrite("~p = ~p\n", [Name,Def]),
            parse_defs(Ifile, nextline(Ifile, L, St), [{Name,Def}|Ms], St);
        _ -> {ok,Line,Ms,St}                    % Anything else
    end;
parse_defs(_, Line, Ms, St) ->
    {ok,Line,Ms,St}.

%% parse_rules(File, Line, Macros, State) -> {ok,NextLine,REAs,Actions,State}.
%%  Parse the RE rules section of the file. This must exist.

parse_rules(Ifile, {ok,?RULE_HEAD ++ Rest,L}, Ms, St) ->
    St1 = warn_ignored_chars(L, Rest, St),
    parse_rules(Ifile, nextline(Ifile, L, St), Ms, [], [], 0, St1);
parse_rules(_, {ok,_,L}, _, St) ->
    add_error({L,leex,missing_rules}, St);
parse_rules(_, {eof,L}, _, St) ->
    add_error({L,leex,missing_rules}, St).

%% parse_rules(File, Result, Macros, RegExpActions, Actions, Acount, State) ->
%%      {ok,NextCLine,RegExpActions,Actions,NewState} | throw(NewState)

parse_rules(Ifile, NextLine, Ms, REAs, As, N, St) ->
    case NextLine of
        {ok,?CODE_HEAD ++ _Rest,_} ->
            parse_rules_end(Ifile, NextLine, REAs, As, St);
        {ok,Chars,L0} ->
            %%io:fwrite("~w: ~p~n", [L0,Chars]),
            case collect_rule(Ifile, Chars, L0) of
                {ok,Re,Atoks,L1} ->
                    {ok,REA,A,St1} = parse_rule(Re, L0, Atoks, Ms, N, St),
                    parse_rules(Ifile, nextline(Ifile, L1, St), Ms,
                                [REA|REAs], [A|As], N+1, St1);
                {error,E} -> add_error(E, St)
            end;
        {eof,_} ->
            parse_rules_end(Ifile, NextLine, REAs, As, St)
    end.

parse_rules_end(_, {ok,_,L}, [], [], St) ->
    add_error({L,leex,empty_rules}, St);
parse_rules_end(_, {eof,L}, [], [], St) ->
    add_error({L,leex,empty_rules}, St);
parse_rules_end(_, NextLine, REAs, As, St) ->
    %% Must be *VERY* careful to put rules in correct order!
    {ok,NextLine,reverse(REAs),reverse(As),St}.

%% collect_rule(File, Line, Lineno) ->
%%      {ok,RegExp,ActionTokens,NewLineno} | {error,E}.
%% Collect a complete rule by reading lines until the the regexp and
%% action has been read. Keep track of line number.

collect_rule(Ifile, Chars, L0) ->
    %% Erlang strings are 1 based, but re 0 :-(
    {match,[{St0,Len}|_]} = re:run(Chars, "[^ \t\r\n]+"),
    St = St0 + 1,
    %%io:fwrite("RE = ~p~n", [substr(Chars, St, Len)]),
    case collect_action(Ifile, substr(Chars, St+Len), L0, []) of
        {ok,[{':',_}|Toks],L1} -> {ok,substr(Chars, St, Len),Toks,L1};
        {ok,_,_} -> {error,{L0,leex,bad_rule}};
        {eof,L1} -> {error,{L1,leex,bad_rule}};
        {error,E,_} -> {error,E}
    end.

collect_action(_Ifile, {error, _}, L, _Cont0) ->
    {error, {L, leex, cannot_parse}, ignored_end_line};
collect_action(Ifile, Chars, L0, Cont0) ->
    case erl_scan:tokens(Cont0, Chars, L0) of
        {done,{ok,Toks,_},_} -> {ok,Toks,L0};
        {done,{eof,_},_} -> {eof,L0};
        {done,{error,E,_},_} -> {error,E,L0};
        {more,Cont1} ->
            collect_action(Ifile, io:get_line(Ifile, leex), L0+1, Cont1)
    end.

%% parse_rule(RegExpString, RegExpLine, ActionTokens, Macros, Counter, State) ->
%%      {ok,{RE,Action},ActionData,State}.
%%  Parse one regexp after performing macro substition.

parse_rule(S, Line, [{dot,_}], Ms, N, St) ->
    case parse_rule_regexp(S, Ms, St) of
        {ok,R} ->
            {ok,{R,N},{N,empty_action},St};
        {error,E} ->
            add_error({Line,leex,E}, St)
    end;
parse_rule(S, Line, Atoks, Ms, N, St) ->
    case parse_rule_regexp(S, Ms, St) of
        {ok,R} ->
            %%io:fwrite("RE = ~p~n", [R]),
            %% Check for token variables.
            TokenChars = var_used('TokenChars', Atoks),
            TokenLen = var_used('TokenLen', Atoks),
            TokenLine = var_used('TokenLine', Atoks),
            {ok,{R,N},{N,Atoks,TokenChars,TokenLen,TokenLine},St};
        {error,E} ->
            add_error({Line,leex,E}, St)
    end.

var_used(Name, Toks) ->
    case lists:keyfind(Name, 3, Toks) of
        {var,_,Name} -> true;                   %It's the var we want
        _ -> false
    end.

%% parse_rule_regexp(RegExpString, Macros, State) ->
%%     {ok,RegExp} | {error,Error}.
%% Substitute in macros and parse RegExpString. Cannot use re:replace
%% here as it uses info in replace string (&).

parse_rule_regexp(RE0, [{M,Exp}|Ms], St) ->
    Split= re:split(RE0, "\\{" ++ M ++ "\\}", [{return,list}]),
    RE1 = string:join(Split, Exp),
    parse_rule_regexp(RE1, Ms, St);
parse_rule_regexp(RE, [], St) ->
    %%io:fwrite("RE = ~p~n", [RE]),
    case re_parse(RE, St) of
        {ok,R} -> {ok,R};
        {error,E} -> {error,{regexp,E}}
    end.

%% parse_code(File, Line, State) -> {ok,Code,NewState}.<commit_msg>Missed a few similar calls<commit_after>%% Copyright (c) 2008,2009 <NAME>. All rights reserved.
%%
%% Redistribution and use in source and binary forms, with or without
%% modification, are permitted provided that the following conditions
%% are met:
%%
%% 1. Redistributions of source code must retain the above copyright
%% notice, this list of conditions and the following disclaimer.
%% 2. Redistributions in binary form must reproduce the above copyright
%% notice, this list of conditions and the following disclaimer in the
%% documentation and/or other materials provided with the distribution.
%%
%% THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
%% "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
%% LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
%% FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
%% COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
%% INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
%% BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
%% LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
%% CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
%% LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
%% ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
%% POSSIBILITY OF SUCH DAMAGE.

%%% A Lexical Analyser Generator for Erlang.
%%%
%%% Most of the algorithms used here are taken pretty much as
%%% described in the "Dragon Book" by Aho, Sethi and Ullman. Some
%%% completing details were taken from "Compiler Design in C" by
%%% Hollub.

-module(leex).

-export([compile/3,file/1,file/2,format_error/1]).

-import(lists, [member/2,reverse/1,sort/1,delete/2,
                keysort/2,keydelete/3,
                map/2,foldl/3,foreach/2,flatmap/2]).
-import(string, [substr/2,substr/3,span/2]).
-import(ordsets, [is_element/2,add_element/2,union/2]).
-import(orddict, [store/3]).

-include("erl_compile.hrl").
%%-include_lib("stdlib/include/erl_compile.hrl").

-define(LEEXINC, "leexinc.hrl").        % Include file
-define(LEEXLIB, parsetools).           % Leex is in lib parsetools
%%-define(LEEXLIB, leex).               % Leex is in lib leex

-define(DEFS_HEAD, "Definitions.").
-define(RULE_HEAD, "Rules.").
-define(CODE_HEAD, "Erlang code.").

-record(leex, {xfile=[],        % Xrl file
               efile=[],        % Erl file
               ifile=[],        % Include file
               gfile=[],        % Graph file
               module,          % Module name
               opts=[],         % Options
               encoding=none,   % Encoding of Xrl file
               % posix=false,   % POSIX regular expressions
               errors=[],
               warnings=[]
              }).

-record(nfa_state, {no,edges=[],accept=noaccept}).
-record(dfa_state, {no,nfa=[],trans=[],accept=noaccept}).

%%%
%%% Exported functions
%%%

%%% Interface to erl_compile.

compile(Input0, Output0,
        #options{warning = WarnLevel, verbose=Verbose, includes=Includes,
		 specific=Specific}) ->
    Input = assure_extension(shorten_filename(Input0), ".xrl"),
    Output = assure_extension(shorten_filename(Output0), ".erl"),
    Includefile = lists:sublist(Includes, 1),
    Werror = proplists:get_bool(warnings_as_errors, Specific),
    Opts = [{scannerfile,Output},{includefile,Includefile},{verbose,Verbose},
            {report_errors,true},{report_warnings,WarnLevel > 0},
	    {warnings_as_errors, Werror}],
    case file(Input, Opts) of
        {ok, _} ->
            ok;
        error ->
            error
    end.

%% file(File) -> ok | error.
%% file(File, Options) -> ok | error.

file(File) -> file(File, []).

file(File, Opts0) ->
    case is_filename(File) of
        no -> erlang:error(badarg, [File,Opts0]);
        _ -> ok
    end,
    Opts = case options(Opts0) of
               badarg ->
                   erlang:error(badarg, [File,Opts0]);
               Options ->
                   Options
           end,
    St0 = #leex{},
    St1 = filenames(File, Opts, St0),   % Get all the filenames
    St = try
             {ok,REAs,Actions,Code,St2} = parse_file(St1),
             {DFA,DF} = make_dfa(REAs, St2),
             case werror(St2) of
                 false ->
                     St3 = out_file(St2, DFA, DF, Actions, Code),
                     case lists:member(dfa_graph, St3#leex.opts) of
                         true -> out_dfa_graph(St3, DFA, DF);
                         false -> St3
                     end;
                 true ->
                     St2
             end
         catch #leex{}=St4 ->
             St4
         end,
    leex_ret(St).             

format_error({file_error, Reason}) ->
    io_lib:fwrite("~ts",[file:format_error(Reason)]);
format_error(missing_defs) -> "missing Definitions";
format_error(missing_rules) -> "missing Rules";
format_error(missing_code) -> "missing Erlang code";
format_error(empty_rules) -> "no rules";
format_error(bad_rule) -> "bad rule";
format_error({regexp,E})->
    Es = case E of
             {interval_range,_} -> "interval range";
             {unterminated,Cs} ->
                 "unterminated " ++ Cs;
             {illegal_char,Cs} ->
                 "illegal character " ++ Cs;
%%           {posix_cc,What} ->
%%               ["illegal POSIX character class ",io_lib:write_string(What)];
             {char_class,What} ->
                 ["illegal character class ",io_lib:write_string(What)]
         end,
    ["bad regexp `",Es,"'"];
format_error(ignored_characters) ->
    "ignored characters";
format_error(cannot_parse) ->
    io_lib:fwrite("cannot parse; probably encoding mismatch", []).

%%%
%%% Local functions
%%%

assure_extension(File, Ext) ->
    lists:concat([strip_extension(File, Ext), Ext]).

%% Assumes File is a filename.
strip_extension(File, Ext) ->
    case filename:extension(File) of
        Ext -> filename:rootname(File);
        _Other -> File
    end.

options(Options0) when is_list(Options0) ->
    try 
        Options = flatmap(fun(return) -> short_option(return, true);
                             (report) -> short_option(report, true);
                             ({return,T}) -> short_option(return, T);
                             ({report,T}) -> short_option(report, T);
                             (T) -> [T]
                          end, Options0),
        options(Options, [scannerfile,includefile,report_errors,
                          report_warnings,warnings_as_errors,
                          return_errors,return_warnings,
                          verbose,dfa_graph], [])
    catch error: _ -> badarg
    end;
options(Option) ->
    options([Option]).

short_option(return, T) ->
    [{return_errors,T}, {return_warnings,T}];
short_option(report, T) ->
    [{report_errors,T}, {report_warnings,T}].

options(Options0, [Key|Keys], L) when is_list(Options0) ->
    Options = case member(Key, Options0) of
                  true -> 
                      [atom_option(Key)|delete(Key, Options0)];
                  false ->
                      Options0
              end,
    V = case lists:keyfind(Key, 1, Options) of
            {Key, Filename0} when Key =:= includefile;
				  Key =:= scannerfile ->
                case is_filename(Filename0) of
                    no -> 
                        badarg;
                    Filename -> 
                        {ok,[{Key,Filename}]}
                end;
            {Key, Bool} = KB when is_boolean(Bool) ->
                {ok, [KB]};
            {Key, _} ->
                badarg;
            false ->
                {ok,[{Key,default_option(Key)}]}
        end,
    case V of
        badarg ->
            badarg;
        {ok,KeyValueL} ->
            NewOptions = keydelete(Key, 1, Options),
            options(NewOptions, Keys, KeyValueL ++ L)
    end;
options([], [], L) ->
    foldl(fun({_,false}, A) -> A;
             ({Tag,true}, A) -> [Tag|A];
             (F,A) -> [F|A]
          end, [], L);
options(_Options, _, _L) ->
    badarg.

default_option(dfa_graph) -> false;
default_option(includefile) -> [];
default_option(report_errors) -> true;
default_option(report_warnings) -> true;
default_option(warnings_as_errors) -> false;
default_option(return_errors) -> false;
default_option(return_warnings) -> false;
default_option(scannerfile) -> [];
default_option(verbose) -> false.

atom_option(dfa_graph) -> {dfa_graph,true};
atom_option(report_errors) -> {report_errors,true};
atom_option(report_warnings) -> {report_warnings,true};
atom_option(warnings_as_errors) -> {warnings_as_errors,true};
atom_option(return_errors) -> {return_errors,true};
atom_option(return_warnings) -> {return_warnings,true};
atom_option(verbose) -> {verbose,true};
atom_option(Key) -> Key.

is_filename(T) ->
    try filename:flatten(T)
    catch error: _ -> no
    end.    

shorten_filename(Name0) ->
    {ok,Cwd} = file:get_cwd(),
    case lists:prefix(Cwd, Name0) of
        false -> Name0;
        true ->
            case lists:nthtail(length(Cwd), Name0) of
                "/"++N -> N;
                N -> N
            end
    end.

leex_ret(St) ->
    report_errors(St),
    report_warnings(St),
    Es = pack_errors(St#leex.errors),
    Ws = pack_warnings(St#leex.warnings),
    Werror = werror(St),
    if 
        Werror ->
            do_error_return(St, Es, Ws);
        Es =:= [] -> 
            case member(return_warnings, St#leex.opts) of
                true -> {ok, St#leex.efile, Ws};
                false -> {ok, St#leex.efile}
            end;
        true ->
            do_error_return(St, Es, Ws)
    end.

do_error_return(St, Es, Ws) ->
    case member(return_errors, St#leex.opts) of
        true -> {error, Es, Ws};
        false -> error
    end.

werror(St) ->
    St#leex.warnings =/= []
	andalso member(warnings_as_errors, St#leex.opts).

pack_errors([{File,_} | _] = Es) ->
    [{File, flatmap(fun({_,E}) -> [E] end, sort(Es))}];
pack_errors([]) ->
    [].
    
pack_warnings([{File,_} | _] = Ws) ->
    [{File, flatmap(fun({_,W}) -> [W] end, sort(Ws))}];
pack_warnings([]) ->
    [].

report_errors(St) ->
    when_opt(fun () -> 
                     foreach(fun({File,{none,Mod,E}}) -> 
                                     io:fwrite("~ts: ~ts\n",
                                               [File,Mod:format_error(E)]);
                                ({File,{Line,Mod,E}}) -> 
                                     io:fwrite("~ts:~w: ~ts\n",
                                               [File,Line,Mod:format_error(E)])
                             end, sort(St#leex.errors))
             end, report_errors, St#leex.opts).

report_warnings(St) ->
    Werror = member(warnings_as_errors, St#leex.opts),
    Prefix = case Werror of
		 true -> "";
		 false -> "Warning: "
	     end,
    ReportWerror = Werror andalso member(report_errors, St#leex.opts),
    ShouldReport = member(report_warnings, St#leex.opts) orelse ReportWerror,
    when_bool(fun () ->
		      foreach(fun({File,{none,Mod,W}}) ->
				      io:fwrite("~ts: ~s~ts\n",
						[File,Prefix,
						 Mod:format_error(W)]);
				 ({File,{Line,Mod,W}}) ->
				      io:fwrite("~ts:~w: ~s~ts\n",
						[File,Line,Prefix,
						 Mod:format_error(W)])
			      end, sort(St#leex.warnings))
	      end, ShouldReport).

-spec add_error(_, #leex{}) -> no_return().
add_error(E, St) ->
    add_error(St#leex.xfile, E, St).

add_error(File, Error, St) ->
    throw(St#leex{errors = [{File,Error}|St#leex.errors]}).

add_warning(Line, W, St) ->
    St#leex{warnings = [{St#leex.xfile,{Line,leex,W}}|St#leex.warnings]}.

%% filenames(File, Options, State) -> State.
%%  The default output dir is the current directory unless an
%%  explicit one has been given in the options.

filenames(File, Opts, St0) ->
    Dir = filename:dirname(File),
    Base = filename:basename(File, ".xrl"),
    Xfile = filename:join(Dir, Base ++ ".xrl"),
    Efile = Base ++ ".erl",
    Gfile = Base ++ ".dot",
    Module = list_to_atom(Base),
    St1 = St0#leex{xfile=Xfile,
                   opts=Opts,
                   module=Module},
    {includefile,Ifile0} = lists:keyfind(includefile, 1, Opts),
    Ifile = inc_file_name(Ifile0),
    %% Test for explicit scanner file.
    {scannerfile,Ofile} = lists:keyfind(scannerfile, 1, Opts),
    if
        Ofile =:= [] ->
            St1#leex{efile=filename:join(Dir, Efile),
                     ifile=Ifile,
                     gfile=filename:join(Dir, Gfile)};
        true ->
            D = filename:dirname(Ofile),
            St1#leex{efile=Ofile,
                     ifile=Ifile,
                     gfile=filename:join(D, Gfile)}
    end.

when_opt(Do, Opt, Opts) ->
    case member(Opt, Opts) of
        true -> Do();
        false -> ok
    end.

when_bool(Do, Bool) ->
    case Bool of
	true -> Do();
	false -> ok
    end.

verbose_print(St, Format, Args) ->
    when_opt(fun () -> io:fwrite(Format, Args) end, verbose, St#leex.opts).

%% parse_file(State) -> {ok,[REA],[Action],Code,NewState} | throw(NewState)
%%  when
%%      REA = {RegExp,ActionNo};
%%      Action = {ActionNo,ActionString};
%%      Code = {StartLine, StartPos, NumOfLines}. Where the Erlang code is.
%%
%%  Read and parse the file Xfile.
%%  After each section of the file has been parsed we directly call the
%%  next section. This is done when we detect a line we don't recognise
%%  in the current section. The file format is very simple and Erlang
%%  token based, we allow empty lines and Erlang style comments.

parse_file(St0) ->
    case file:open(St0#leex.xfile, [read]) of
        {ok,Xfile} ->
            St1 = St0#leex{encoding = epp:set_encoding(Xfile)},
            try
                verbose_print(St1, "Parsing file ~ts, ", [St1#leex.xfile]),
                %% We KNOW that errors throw so we can ignore them here.
                {ok,Line1,St2} = parse_head(Xfile, St1),
                {ok,Line2,Macs,St3} = parse_defs(Xfile, Line1, St2),
                {ok,Line3,REAs,Actions,St4} =
                    parse_rules(Xfile, Line2, Macs, St3),
                {ok,Code,St5} = parse_code(Xfile, Line3, St4),
                verbose_print(St5, "contained ~w rules.~n", [length(REAs)]),
                {ok,REAs,Actions,Code,St5}
            after ok = file:close(Xfile)
            end;
        {error,Error} ->
            add_error({none,leex,{file_error,Error}}, St0)
    end.

%% parse_head(File, State) -> {ok,NextLine,State}.
%%  Parse the head of the file. Skip all comments and blank lines.

parse_head(Ifile, St) -> {ok,nextline(Ifile, 0, St),St}.

%% parse_defs(File, Line, State) -> {ok,NextLine,Macros,State}.
%%  Parse the macro definition section of a file. This must exist.
%%  The section is ended by a non-blank line which is not a macro def.

parse_defs(Ifile, {ok,?DEFS_HEAD ++ Rest,L}, St) ->
    St1 = warn_ignored_chars(L, Rest, St),
    parse_defs(Ifile, nextline(Ifile, L, St), [], St1);
parse_defs(_, {ok,_,L}, St) ->
    add_error({L,leex,missing_defs}, St);
parse_defs(_, {eof,L}, St) ->
    add_error({L,leex,missing_defs}, St).

parse_defs(Ifile, {ok,Chars,L}=Line, Ms, St) ->
    %% This little beauty matches out a macro definition, RE's are so clear.
    MS = "^[ \t]*([A-Z_][A-Za-z0-9_]*)[ \t]*=[ \t]*([^ \t\r\n]*)[ \t\r\n]*\$",
    case re:run(Chars, MS, [{capture,all_but_first,list},unicode]) of
        {match,[Name,Def]} ->
            %%io:fwrite("~p = ~p\n", [Name,Def]),
            parse_defs(Ifile, nextline(Ifile, L, St), [{Name,Def}|Ms], St);
        _ -> {ok,Line,Ms,St}                    % Anything else
    end;
parse_defs(_, Line, Ms, St) ->
    {ok,Line,Ms,St}.

%% parse_rules(File, Line, Macros, State) -> {ok,NextLine,REAs,Actions,State}.
%%  Parse the RE rules section of the file. This must exist.

parse_rules(Ifile, {ok,?RULE_HEAD ++ Rest,L}, Ms, St) ->
    St1 = warn_ignored_chars(L, Rest, St),
    parse_rules(Ifile, nextline(Ifile, L, St), Ms, [], [], 0, St1);
parse_rules(_, {ok,_,L}, _, St) ->
    add_error({L,leex,missing_rules}, St);
parse_rules(_, {eof,L}, _, St) ->
    add_error({L,leex,missing_rules}, St).

%% parse_rules(File, Result, Macros, RegExpActions, Actions, Acount, State) ->
%%      {ok,NextCLine,RegExpActions,Actions,NewState} | throw(NewState)

parse_rules(Ifile, NextLine, Ms, REAs, As, N, St) ->
    case NextLine of
        {ok,?CODE_HEAD ++ _Rest,_} ->
            parse_rules_end(Ifile, NextLine, REAs, As, St);
        {ok,Chars,L0} ->
            %%io:fwrite("~w: ~p~n", [L0,Chars]),
            case collect_rule(Ifile, Chars, L0) of
                {ok,Re,Atoks,L1} ->
                    {ok,REA,A,St1} = parse_rule(Re, L0, Atoks, Ms, N, St),
                    parse_rules(Ifile, nextline(Ifile, L1, St), Ms,
                                [REA|REAs], [A|As], N+1, St1);
                {error,E} -> add_error(E, St)
            end;
        {eof,_} ->
            parse_rules_end(Ifile, NextLine, REAs, As, St)
    end.

parse_rules_end(_, {ok,_,L}, [], [], St) ->
    add_error({L,leex,empty_rules}, St);
parse_rules_end(_, {eof,L}, [], [], St) ->
    add_error({L,leex,empty_rules}, St);
parse_rules_end(_, NextLine, REAs, As, St) ->
    %% Must be *VERY* careful to put rules in correct order!
    {ok,NextLine,reverse(REAs),reverse(As),St}.

%% collect_rule(File, Line, Lineno) ->
%%      {ok,RegExp,ActionTokens,NewLineno} | {error,E}.
%% Collect a complete rule by reading lines until the the regexp and
%% action has been read. Keep track of line number.

collect_rule(Ifile, Chars, L0) ->
    %% Erlang strings are 1 based, but re 0 :-(
    {match,[{St0,Len}|_]} = re:run(Chars, "[^ \t\r\n]+", [unicode]),
    St = St0 + 1,
    %%io:fwrite("RE = ~p~n", [substr(Chars, St, Len)]),
    case collect_action(Ifile, substr(Chars, St+Len), L0, []) of
        {ok,[{':',_}|Toks],L1} -> {ok,substr(Chars, St, Len),Toks,L1};
        {ok,_,_} -> {error,{L0,leex,bad_rule}};
        {eof,L1} -> {error,{L1,leex,bad_rule}};
        {error,E,_} -> {error,E}
    end.

collect_action(_Ifile, {error, _}, L, _Cont0) ->
    {error, {L, leex, cannot_parse}, ignored_end_line};
collect_action(Ifile, Chars, L0, Cont0) ->
    case erl_scan:tokens(Cont0, Chars, L0) of
        {done,{ok,Toks,_},_} -> {ok,Toks,L0};
        {done,{eof,_},_} -> {eof,L0};
        {done,{error,E,_},_} -> {error,E,L0};
        {more,Cont1} ->
            collect_action(Ifile, io:get_line(Ifile, leex), L0+1, Cont1)
    end.

%% parse_rule(RegExpString, RegExpLine, ActionTokens, Macros, Counter, State) ->
%%      {ok,{RE,Action},ActionData,State}.
%%  Parse one regexp after performing macro substition.

parse_rule(S, Line, [{dot,_}], Ms, N, St) ->
    case parse_rule_regexp(S, Ms, St) of
        {ok,R} ->
            {ok,{R,N},{N,empty_action},St};
        {error,E} ->
            add_error({Line,leex,E}, St)
    end;
parse_rule(S, Line, Atoks, Ms, N, St) ->
    case parse_rule_regexp(S, Ms, St) of
        {ok,R} ->
            %%io:fwrite("RE = ~p~n", [R]),
            %% Check for token variables.
            TokenChars = var_used('TokenChars', Atoks),
            TokenLen = var_used('TokenLen', Atoks),
            TokenLine = var_used('TokenLine', Atoks),
            {ok,{R,N},{N,Atoks,TokenChars,TokenLen,TokenLine},St};
        {error,E} ->
            add_error({Line,leex,E}, St)
    end.

var_used(Name, Toks) ->
    case lists:keyfind(Name, 3, Toks) of
        {var,_,Name} -> true;                   %It's the var we want
        _ -> false
    end.

%% parse_rule_regexp(RegExpString, Macros, State) ->
%%     {ok,RegExp} | {error,Error}.
%% Substitute in macros and parse RegExpString. Cannot use re:replace
%% here as it uses info in replace string (&).

parse_rule_regexp(RE0, [{M,Exp}|Ms], St) ->
    Split= re:split(RE0, "\\{" ++ M ++ "\\}", [{return,list},unicode]),
    RE1 = string:join(Split, Exp),
    parse_rule_regexp(RE1, Ms, St);
parse_rule_regexp(RE, [], St) ->
    %%io:fwrite("RE = ~p~n", [RE]),
    case re_parse(RE, St) of
        {ok,R} -> {ok,R};
        {error,E} -> {error,{regexp,E}}
    end.

%% parse_code(File, Line, State) -> {ok,Code,NewState}.
{'fraction_non_alphanumeric': 0.15752062829307087, 'fraction_numerical': 0.007456009543692216, 'mean_word_length': 2.3995437647854003, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 371, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5295484', 'n_tokens_mistral': 14044, 'n_tokens_neox': 13503, 'n_words': 3058}