<commit_before>%%%----------------------------------------------------------------------
%%% File        : instr.erl
%%% Authors     : <NAME> <<EMAIL>>
%%%               <NAME> <<EMAIL>>
%%% Description : Instrumenter
%%% Created     : 16 May 2010
%%%
%%% @doc: Instrumenter
%%% @end
%%%----------------------------------------------------------------------

-module(instr).
-export([delete_and_purge/1, instrument_and_load/1]).

-include("gen.hrl").

-define(INCLUDE_DIR, filename:absname("include")).

%% Instrumented auto-imported functions of 'erlang' module.
-define(INSTR_ERLANG_NO_MOD,
	[demonitor, halt, link, monitor, process_flag, register, spawn,
	 spawn_link, spawn_monitor, unlink, unregister, whereis]).

%% Instrumented functions called as erlang:FUNCTION.
-define(INSTR_ERLANG,
	[demonitor, halt, link, monitor, process_flag, register, send,
	 spawn, spawn_link, spawn_monitor, unlink, unregister, whereis,
	 yield]).

%% Delete and purge all modules in Files.
-spec delete_and_purge([file()]) -> 'ok'.

delete_and_purge(Files) ->
    ModsToPurge = [list_to_atom(filename:basename(F, ".erl")) || F <- Files],
    [begin code:delete(M), code:purge(M) end || M <- ModsToPurge],
    ok.

%% @spec instrument_and_load(Files::[file()]) -> 'ok' | 'error'
%% @doc: Instrument, compile and load a list of files.
%%
%% Each file is first validated (i.e. checked whether it will compile
%% successfully). If no errors are encountered, the file gets instrumented,
%% compiled and loaded. If all of these actions are successfull, the function
%% returns `ok', otherwise `error' is returned. No `.beam' files are produced.
-spec instrument_and_load([file()]) -> 'ok' | 'error'.

instrument_and_load([]) ->
    log:log("No files instrumented~n"),
    ok;
instrument_and_load([File]) ->
    instrument_and_load_one(File);
instrument_and_load([File | Rest]) ->
    case instrument_and_load_one(File) of
	ok -> instrument_and_load(Rest);
	error -> error
    end.

%% Instrument and load a single file.
instrument_and_load_one(File) ->
    %% Compilation of original file without emitting code, just to show
    %% warnings or stop if an error is found, before instrumenting it.
    log:log("Validating file ~p...~n", [File]),
    PreOptions = [strong_validation, verbose, return, {i, ?INCLUDE_DIR}],
    case compile:file(File, PreOptions) of
	{ok, Module, Warnings} ->
	    %% Log warning messages.
	    log_warning_list(Warnings),
	    %% A table for holding used variable names.
	    ets:new(?NT_USED, [named_table, private]),
	    %% Instrument given source file.
	    log:log("Instrumenting file ~p...~n", [File]),
	    case instrument(File) of
		{ok, NewForms} ->
		    %% Delete `used` table.
		    ets:delete(?NT_USED),
		    %% Compile instrumented code.
		    %% TODO: More compile options?
		    log:log("Compiling instrumented code...~n"),
		    CompOptions = [binary],
		    case compile:forms(NewForms, CompOptions) of
			{ok, Module, Binary} ->
			    log:log("Loading module `~p`...~n", [Module]),
			    case code:load_binary(Module, File, Binary) of
				{module, Module} ->
				    ok;
				{error, Error} ->
				    log:log("error~n~p~n", [Error]),
				    error
			    end;
			error ->
			    log:log("error~n"),
			    error
		    end;
		{error, Error} ->
		    log:log("error: ~p~n", [Error]),
		    %% Delete `used` table.
		    ets:delete(?NT_USED),
		    error
	    end;
	{error, Errors, Warnings} ->
	    log_error_list(Errors),
	    log_warning_list(Warnings),
	    log:log("error~n"),
	    error
    end.

instrument(File) ->
    %% TODO: For now using the default and the test directory include path.
    %%       In the future we have to provide a means for an externally
    %%       defined include path (like the erlc -I flag).
    case epp:parse_file(File, [?INCLUDE_DIR, filename:dirname(File)], []) of
	{ok, OldForms} ->
	    Tree = erl_recomment:recomment_forms(OldForms, []),
	    MapFun = fun(T) -> instrument_toplevel(T) end,
	    Transformed = erl_syntax_lib:map_subtrees(MapFun, Tree),
	    Abstract = erl_syntax:revert(Transformed),
	    %% io:put_chars(erl_prettypr:format(Abstract)),
	    NewForms = erl_syntax:form_list_elements(Abstract),
	    {ok, NewForms};
	{error, Error} -> {error, Error}
    end.

%% Instrument a "top-level" element.
%% Of the "top-level" elements, i.e. functions, specs, etc., only functions are
%% transformed, so leave everything else as is.
instrument_toplevel(Tree) ->
    case erl_syntax:type(Tree) of
	function -> instrument_function(Tree);
	_Other -> Tree
    end.

%% Instrument a function.
instrument_function(Tree) ->
    %% Delete previous entry in `used` table (if any).
    ets:delete_all_objects(?NT_USED),
    %% A set of all variables used in the function.
    Used = erl_syntax_lib:variables(Tree),
    %% Insert the used set into `used` table.
    ets:insert(?NT_USED, {used, Used}),
    instrument_subtrees(Tree).

%% Instrument all subtrees of Tree.
instrument_subtrees(Tree) ->
    MapFun = fun(T) -> instrument_term(T) end,
    erl_syntax_lib:map_subtrees(MapFun, Tree).

%% Instrument a term.
instrument_term(Tree) ->
    case erl_syntax:type(Tree) of
	application ->
	    NewTree = instrument_subtrees(Tree),
	    case get_mfa(NewTree) of
		no_instr -> NewTree;
		Mfa -> instrument_application(Mfa)
	    end;
	infix_expr ->
	    Operator = erl_syntax:infix_expr_operator(Tree),
	    case erl_syntax:operator_name(Operator) of
		'!' -> instrument_send(instrument_subtrees(Tree));
		_Other -> instrument_subtrees(Tree)
	    end;
	receive_expr -> instrument_receive(instrument_subtrees(Tree));
	underscore -> new_underscore_variable();
	_Other -> instrument_subtrees(Tree)
    end.

%% Return {ModuleAtom, FunctionAtom, ArgTree} for a function that is going
%% to be instrumented or 'no_instr' otherwise.
get_mfa(Tree) ->
    Qualifier = erl_syntax:application_operator(Tree),
    case erl_syntax:type(Qualifier) of
	atom ->
	    Function = erl_syntax:atom_value(Qualifier),
	    ArgTree = erl_syntax:application_arguments(Tree),
	    case lists:member(Function, ?INSTR_ERLANG_NO_MOD) of
		true -> {erlang, Function, ArgTree};
		false -> no_instr
	    end;
	module_qualifier ->
	    ModTree = erl_syntax:module_qualifier_argument(Qualifier),
	    FunTree = erl_syntax:module_qualifier_body(Qualifier),
	    case erl_syntax:type(ModTree) =:= atom andalso
		erl_syntax:type(FunTree) =:= atom of
		true ->
		    Module = erl_syntax:atom_value(ModTree),
		    Function = erl_syntax:atom_value(FunTree),
		    case Module of
			erlang ->
			    case lists:member(Function, ?INSTR_ERLANG) of
				true ->
				    ArgTree =
					erl_syntax:application_arguments(Tree),
				    {Module, Function, ArgTree};
				false -> no_instr
			    end;
			_Other -> no_instr
		    end;
		false -> no_instr
	    end;
	_Other -> no_instr
    end.

%% Instrument an application (function call).
instrument_application({erlang, Fun, ArgTree}) ->
    ModTree = erl_syntax:atom(sched),
    FunTree = erl_syntax:atom(list_to_atom("rep_" ++ atom_to_list(Fun))),<commit_msg>Perform some refactorings Re #49<commit_after>%%%----------------------------------------------------------------------
%%% File        : instr.erl
%%% Authors     : <NAME> <<EMAIL>>
%%%               <NAME> <<EMAIL>>
%%% Description : Instrumenter
%%% Created     : 16 May 2010
%%%
%%% @doc: Instrumenter
%%% @end
%%%----------------------------------------------------------------------

-module(instr).
-export([delete_and_purge/1, instrument_and_load/1]).

-include("gen.hrl").

-define(INCLUDE_DIR, filename:absname("include")).

%% Instrumented auto-imported functions of 'erlang' module.
-define(INSTR_ERLANG_NO_MOD,
	[demonitor, halt, link, monitor, process_flag, register, spawn,
	 spawn_link, spawn_monitor, unlink, unregister, whereis]).

%% Instrumented functions called as erlang:FUNCTION.
-define(INSTR_ERLANG,
	[send, yield] ++ ?INSTR_ERLANG_NO_MOD).

%% Delete and purge all modules in Files.
-spec delete_and_purge([file()]) -> 'ok'.

delete_and_purge(Files) ->
    ModsToPurge = [list_to_atom(filename:basename(F, ".erl")) || F <- Files],
    [begin code:delete(M), code:purge(M) end || M <- ModsToPurge],
    ok.

%% @spec instrument_and_load(Files::[file()]) -> 'ok' | 'error'
%% @doc: Instrument, compile and load a list of files.
%%
%% Each file is first validated (i.e. checked whether it will compile
%% successfully). If no errors are encountered, the file gets instrumented,
%% compiled and loaded. If all of these actions are successfull, the function
%% returns `ok', otherwise `error' is returned. No `.beam' files are produced.
-spec instrument_and_load([file()]) -> 'ok' | 'error'.

instrument_and_load([]) ->
    log:log("No files instrumented~n"),
    ok;
instrument_and_load([File]) ->
    instrument_and_load_one(File);
instrument_and_load([File | Rest]) ->
    case instrument_and_load_one(File) of
	ok -> instrument_and_load(Rest);
	error -> error
    end.

%% Instrument and load a single file.
instrument_and_load_one(File) ->
    %% Compilation of original file without emitting code, just to show
    %% warnings or stop if an error is found, before instrumenting it.
    log:log("Validating file ~p...~n", [File]),
    PreOptions = [strong_validation, verbose, return, {i, ?INCLUDE_DIR}],
    case compile:file(File, PreOptions) of
	{ok, Module, Warnings} ->
	    %% Log warning messages.
	    log_warning_list(Warnings),
	    %% A table for holding used variable names.
	    ets:new(?NT_USED, [named_table, private]),
	    %% Instrument given source file.
	    log:log("Instrumenting file ~p...~n", [File]),
	    case instrument(File) of
		{ok, NewForms} ->
		    %% Delete `used` table.
		    ets:delete(?NT_USED),
		    %% Compile instrumented code.
		    %% TODO: More compile options?
		    log:log("Compiling instrumented code...~n"),
		    CompOptions = [binary],
		    case compile:forms(NewForms, CompOptions) of
			{ok, Module, Binary} ->
			    log:log("Loading module `~p`...~n", [Module]),
			    case code:load_binary(Module, File, Binary) of
				{module, Module} ->
				    ok;
				{error, Error} ->
				    log:log("error~n~p~n", [Error]),
				    error
			    end;
			error ->
			    log:log("error~n"),
			    error
		    end;
		{error, Error} ->
		    log:log("error: ~p~n", [Error]),
		    %% Delete `used` table.
		    ets:delete(?NT_USED),
		    error
	    end;
	{error, Errors, Warnings} ->
	    log_error_list(Errors),
	    log_warning_list(Warnings),
	    log:log("error~n"),
	    error
    end.

instrument(File) ->
    %% TODO: For now using the default and the test directory include path.
    %%       In the future we have to provide a means for an externally
    %%       defined include path (like the erlc -I flag).
    case epp:parse_file(File, [?INCLUDE_DIR, filename:dirname(File)], []) of
	{ok, OldForms} ->
	    Tree = erl_recomment:recomment_forms(OldForms, []),
	    MapFun = fun(T) -> instrument_toplevel(T) end,
	    Transformed = erl_syntax_lib:map_subtrees(MapFun, Tree),
	    Abstract = erl_syntax:revert(Transformed),
	    %% io:put_chars(erl_prettypr:format(Abstract)),
	    NewForms = erl_syntax:form_list_elements(Abstract),
	    {ok, NewForms};
	{error, Error} -> {error, Error}
    end.

%% Instrument a "top-level" element.
%% Of the "top-level" elements, i.e. functions, specs, etc., only functions are
%% transformed, so leave everything else as is.
instrument_toplevel(Tree) ->
    case erl_syntax:type(Tree) of
	function -> instrument_function(Tree);
	_Other -> Tree
    end.

%% Instrument a function.
instrument_function(Tree) ->
    %% Delete previous entry in `used` table (if any).
    ets:delete_all_objects(?NT_USED),
    %% A set of all variables used in the function.
    Used = erl_syntax_lib:variables(Tree),
    %% Insert the used set into `used` table.
    ets:insert(?NT_USED, {used, Used}),
    instrument_subtrees(Tree).

%% Instrument all subtrees of Tree.
instrument_subtrees(Tree) ->
    MapFun = fun(T) -> instrument_term(T) end,
    erl_syntax_lib:map_subtrees(MapFun, Tree).

%% Instrument a term.
instrument_term(Tree) ->
    case erl_syntax:type(Tree) of
	application ->
	    NewTree = instrument_subtrees(Tree),
	    case get_mfa(NewTree) of
		no_instr -> NewTree;
		Mfa -> instrument_application(Mfa)
	    end;
	infix_expr ->
	    Operator = erl_syntax:infix_expr_operator(Tree),
	    case erl_syntax:operator_name(Operator) of
		'!' -> instrument_send(instrument_subtrees(Tree));
		_Other -> instrument_subtrees(Tree)
	    end;
	receive_expr -> instrument_receive(instrument_subtrees(Tree));
	underscore -> new_underscore_variable();
	_Other -> instrument_subtrees(Tree)
    end.

%% Return {ModuleAtom, FunctionAtom, ArgTree} for a function that is going
%% to be instrumented or 'no_instr' otherwise.
get_mfa(Tree) ->
    Qualifier = erl_syntax:application_operator(Tree),
    case erl_syntax:type(Qualifier) of
	atom ->
	    Function = erl_syntax:atom_value(Qualifier),
	    case lists:member(Function, ?INSTR_ERLANG_NO_MOD) of
		true ->
                    ArgTree = erl_syntax:application_arguments(Tree),
                    {erlang, Function, ArgTree};
		false -> no_instr
	    end;
	module_qualifier ->
	    ModTree = erl_syntax:module_qualifier_argument(Qualifier),
	    FunTree = erl_syntax:module_qualifier_body(Qualifier),
	    case erl_syntax:type(ModTree) =:= atom andalso
		erl_syntax:type(FunTree) =:= atom of
		true ->
		    Module = erl_syntax:atom_value(ModTree),
		    case Module of
			erlang ->
                            Function = erl_syntax:atom_value(FunTree),
			    case lists:member(Function, ?INSTR_ERLANG) of
				true ->
				    ArgTree =
					erl_syntax:application_arguments(Tree),
				    {Module, Function, ArgTree};
				false -> no_instr
			    end;
			_Other -> no_instr
		    end;
		false -> no_instr
	    end;
	_Other -> no_instr
    end.

%% Instrument an application (function call).
instrument_application({erlang, Fun, ArgTree}) ->
    ModTree = erl_syntax:atom(sched),
    FunTree = erl_syntax:atom(list_to_atom("rep_" ++ atom_to_list(Fun))),
{'fraction_non_alphanumeric': 0.1519185591229444, 'fraction_numerical': 0.0012814124012244608, 'mean_word_length': 3.301285976729945, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 117, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3983880', 'n_tokens_mistral': 4963, 'n_tokens_neox': 4454, 'n_words': 1126}