<commit_before>% Author: <NAME>
% Has the ability to compile Erlang into CoreErlang and also compile
% from Erlang straight to a CoreErlang AST.

-module(coregen).
-export([er2ce/1,
         er2ce/2,
         er2ast/1,
         er2ast/2]).

% Compiles a given Erlang source file to a CoreErlang file
er2ce(Module) ->
    code:add_path("../lib/"),
    er2ce(Module, filepath:path(Module)).

er2ce(Module, OutputDirectory) ->
    case re:run(OutputDirectory, ".*/$") of
        nomatch ->
            {error, output_directory_not_valid};
        _ ->
            code:add_path("../lib/"),
            compile:file(Module, to_core),

            % Compiling always generates output in working directory so lets
            % move it into the directory where source code exists
            FileName = filepath:name(Module),
            filepath:move(FileName ++ ".core", OutputDirectory ++ FileName ++ ".core"),
            {ok, core_compiled}
    end.

% Compiles a given Erlang source file to a CoreErlang AST file
er2ast(Module) ->
    code:add_path("../lib/"),
    er2ast(Module, filepath:path(Module)).

er2ast(Module, OutputDirectory) ->
    code:add_path("../lib/"),
    er2ce(Module, "./"),

    % Parse core erlang and generate AST, credits to
    % http://www.robertjakob.de/posts/ceast.html :)
    % for general method to generate AST
    ModuleName = filepath:name(Module),
    case file:read_file(ModuleName ++ ".core") of
        {ok, Bin} ->
            case core_scan:string(binary_to_list(Bin)) of
                {ok, Toks, _} ->
                    case core_parse:parse(Toks) of
                        {ok, AST} ->
                            % Move file to wherever user specified. This has an aftereffect of ensuring dir exists
                            % so we can write the AST straight to directory
                            filepath:move(ModuleName ++ ".core", OutputDirectory ++ ModuleName ++ ".core"),
                            file:write_file(OutputDirectory ++ ModuleName ++ ".ast", tuple_to_string(AST)),
                            {ok, ast_compiled};
                        {error, E} ->
                            {error, {parse, E}}
                    end;
                {error, E, _} ->
                    {error, {scan, E}}
            end; 
        {error, E} ->          
            {error, {read, E}}
    end.

% Surprisingly, no tuple_to_string functions exist as a BIF
tuple_to_string(T) ->
    lists:flatten(io_lib:format("~p", [T])).<commit_msg>Added AST export variant to er2ast function. This makes it significantly easier to pass into the next function in the transpiler pipeline.<commit_after>% Author: <NAME>
% Has the ability to compile Erlang into CoreErlang and also compile
% from Erlang straight to a CoreErlang AST.

-module(coregen).
-export([er2ce/1,
         er2ce/2,
         er2ast/1,
         er2ast/2]).

% Compiles a given Erlang source file to a CoreErlang file
er2ce(Module) ->
    code:add_path("../lib/"),
    er2ce(Module, filepath:path(Module)).

er2ce(Module, OutputDirectory) ->
    case re:run(OutputDirectory, ".*/$") of
        nomatch ->
            {error, output_directory_not_valid};
        _ ->
            code:add_path("../lib/"),
            compile:file(Module, to_core),

            % Compiling always generates output in working directory so lets
            % move it into the directory where source code exists
            FileName = filepath:name(Module),
            filepath:move(FileName ++ ".core", OutputDirectory ++ FileName ++ ".core"),
            {ok, core_compiled}
    end.

% Compiles a given Erlang source file to a CoreErlang AST file
er2ast(Module) ->
    code:add_path("../lib/"),
    er2ast(Module, filepath:path(Module)).

er2ast(Module, return_AST) ->
    code:add_path("../lib/"),
    er2ce(Module, "./"),

    % Parse core erlang and generate AST, credits to
    % http://www.robertjakob.de/posts/ceast.html :)
    % for general method to generate AST
    ModuleName = filepath:name(Module),
    case file:read_file(ModuleName ++ ".core") of
        {ok, Bin} ->
            case core_scan:string(binary_to_list(Bin)) of
                {ok, Toks, _} ->
                    core_parse:parse(Toks);
                {error, E, _} ->
                    {error, {scan, E}}
            end; 
        {error, E} ->          
            {error, {read, E}}
    end;


er2ast(Module, OutputDirectory) ->
    code:add_path("../lib/"),
    ModuleName = filepath:name(Module),
	case er2ast(Module, return_AST) of
		{ok, AST} ->
			% Move file to wherever user specified. This has an aftereffect of ensuring dir exists
			% so we can write the AST straight to directory
			filepath:move(ModuleName ++ ".core", OutputDirectory ++ ModuleName ++ ".core"),
			file:write_file(OutputDirectory ++ ModuleName ++ ".ast", tuple_to_string(AST)),
			{ok, ast_compiled};
		{error, E} ->
			{error, E}
	end.


% Surprisingly, no tuple_to_string functions exist as a BIF
tuple_to_string(T) ->
    lists:flatten(io_lib:format("~p", [T])).
{'fraction_non_alphanumeric': 0.1257699185376515, 'fraction_numerical': 0.0065567256109676135, 'mean_word_length': 2.178030303030303, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5167948', 'n_tokens_mistral': 1543, 'n_tokens_neox': 1490, 'n_words': 414}