<commit_before>%  @copyright 2010 Konrad-Zuse-Zentrum fuer Informationstechnik Berlin
%  @end
%
%   Licensed under the Apache License, Version 2.0 (the "License");
%   you may not use this file except in compliance with the License.
%   You may obtain a copy of the License at
%
%       http://www.apache.org/licenses/LICENSE-2.0
%
%   Unless required by applicable law or agreed to in writing, software
%   distributed under the License is distributed on an "AS IS" BASIS,
%   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%   See the License for the specific language governing permissions and
%   limitations under the License.
%%%-------------------------------------------------------------------
%%% File    tester.erl
%%% @author <NAME> <<EMAIL>>
%%% @doc    test generator
%%% @end
%%% Created :  30 March 2010 by <NAME> <<EMAIL>>
%%%-------------------------------------------------------------------
%% @version $Id$
-module(tester).

-author('<EMAIL>').
-vsn('$Id$').

-export([test/4]).

% for unit tests
-export([create_value/3]).

-include_lib("tester.hrl").
-include_lib("unittest.hrl").

list_length_min() -> 1.
list_length_max() -> 5.

integer_min() -> 1.
integer_max() -> 5.

-spec test/4 :: (module(), atom(), non_neg_integer(), non_neg_integer()) -> ok.
test(Module, Func, Arity, Iterations) ->
    EmptyParseState = tester_parse_state:new_parse_state(),
    ParseState = collect_fun_info(Module, Func, Arity,
                                  EmptyParseState),
    run(Module, Func, Arity, Iterations, ParseState),
    ok.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% collect necessary type information
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-spec collect_fun_info/4 :: (module(), atom(), non_neg_integer(), #parse_state{}) -> #parse_state{}.
collect_fun_info(Module, Func, Arity, ParseState) ->
    ParseState2 = case tester_parse_state:lookup_type({'fun', Module, Func, Arity}, ParseState) of
                    {value, _} ->
                        ParseState;
                    none ->
                        {ok, {Module, [{abstract_code, {_AbstVersion, AbstractCode}}]}}
                            = beam_lib:chunks(code:where_is_file(atom_to_list(Module) ++ ".beam"), [abstract_code]),
                        lists:foldl(fun (Chunk, InnerParseState) ->
                                            parse_chunk(Chunk, Module, InnerParseState)
                                    end, ParseState, AbstractCode)
                end,
    ParseState3 = case tester_parse_state:get_unknown_types(ParseState2) of
        [] ->
            ParseState2;
        UnknownTypes ->
            collect_unknown_type_infos(UnknownTypes, ParseState2)
    end,
    case tester_parse_state:lookup_type({'fun', Module, Func, Arity}, ParseState3) of
        {value, _} ->
            ParseState3;
        none ->
            ?ct_fail("unknown function ~p:~p/~p~n", [Module, Func, Arity])
    end.

-spec collect_unknown_type_infos/2 :: (list(type_name()), #parse_state{}) -> #parse_state{}.
collect_unknown_type_infos(UnknownTypes, ParseState) ->
    ParseState2 = tester_parse_state:reset_unknown_types(ParseState),
    ParseState3 = lists:foldl(fun ({type, Module, TypeName}, InnerParseState) ->
                                        collect_type_info(Module, TypeName, InnerParseState)
                                end, ParseState2, UnknownTypes),
    case tester_parse_state:get_unknown_types(ParseState3) of
        [] ->
            ParseState3;
        NewUnknownTypes ->
            collect_unknown_type_infos(NewUnknownTypes, ParseState3)
    end.

-spec collect_type_info/3 :: (module(), atom(), #parse_state{}) -> #parse_state{}.
collect_type_info(Module, Type, ParseState) ->
    case tester_parse_state:is_known_type(Module, Type, ParseState) of
        true ->
            ParseState;
        false ->
            {ok, {Module, [{abstract_code, {_AbstVersion, AbstractCode}}]}}
                = beam_lib:chunks(code:where_is_file(atom_to_list(Module) ++ ".beam"), [abstract_code]),
            lists:foldl(fun (Chunk, InnerParseState) ->
                                parse_chunk(Chunk, Module, InnerParseState)
                        end, ParseState, AbstractCode)
    end.


-spec parse_chunk/3 :: (any(), module(), #parse_state{}) -> #parse_state{}.
parse_chunk({attribute, _Line, type, {{record, TypeName}, ATypeSpec, _List}}, Module, ParseState) ->
    {TheTypeSpec, NewParseState} = parse_type(ATypeSpec, Module, ParseState),
    tester_parse_state:add_type_spec({type, Module, TypeName}, TheTypeSpec, NewParseState);
parse_chunk({attribute, _Line, type, {TypeName, ATypeSpec, _List}}, Module, ParseState) ->
    {TheTypeSpec, NewParseState} = parse_type(ATypeSpec, Module, ParseState),
    tester_parse_state:add_type_spec({type, Module, TypeName}, TheTypeSpec, NewParseState);
parse_chunk({attribute, _Line, 'spec', {{FunName, FunArity}, [AFunSpec]}}, Module, ParseState) ->
    {TheFunSpec, NewParseState} = parse_type(AFunSpec, Module, ParseState),
    tester_parse_state:add_type_spec({'fun', Module, FunName, FunArity}, TheFunSpec, NewParseState);
parse_chunk({attribute, _Line, record, {TypeName, TypeList}}, Module, ParseState) ->
    {TheTypeSpec, NewParseState} = parse_type(TypeList, Module, ParseState),
    tester_parse_state:add_type_spec({type, Module, TypeName}, TheTypeSpec, NewParseState);
parse_chunk({attribute, _Line, _AttributeName, _AttributeValue}, _Module, ParseState) ->
    ParseState;
parse_chunk({function, _Line, _FunName, _FunArity, FunCode}, _Module, ParseState) ->
    tester_value_collector:parse_expression(FunCode, ParseState);
parse_chunk({eof, _Line}, _Module, ParseState) ->
    ParseState.

-spec parse_type/3 :: (any(), module(), #parse_state{}) -> {type_spec() , #parse_state{}}.
parse_type({type, _Line, 'fun', [Arg, Result]}, Module, ParseState) ->
    {ArgType, ParseState2} = parse_type(Arg, Module, ParseState),
    {ResultType, ParseState3} = parse_type(Result, Module, ParseState2),
    {{'fun', ArgType, ResultType}, ParseState3};
parse_type({type, _Line, product, Types}, Module, ParseState) ->
    {TypeList, ParseState2} = parse_type_list(Types, Module, ParseState),
    {{product, TypeList}, ParseState2};
parse_type({type, _Line, tuple, any}, _Module, ParseState) ->
    {{tuple, {typedef, tester, test_any}}, ParseState};
parse_type({type, _Line, tuple, Types}, Module, ParseState) ->
    {TypeList, ParseState2} = parse_type_list(Types, Module, ParseState),
    {{tuple, TypeList}, ParseState2};
parse_type({type, _Line, list, [Type]}, Module, ParseState) ->
    {ListType, ParseState2} = parse_type(Type, Module, ParseState),
    {{list, ListType}, ParseState2};
parse_type({type, _Line, nonempty_list, [Type]}, Module, ParseState) ->
    {ListType, ParseState2} = parse_type(Type, Module, ParseState),
    {{nonempty_list, ListType}, ParseState2};
parse_type({type, _Line, list, []}, _Module, ParseState) ->
    {{list, {typedef, tester, test_any}}, ParseState};
parse_type({type, _Line, range, [Begin, End]}, Module, ParseState) ->
    {BeginType, ParseState2} = parse_type(Begin, Module, ParseState),
    {EndType, ParseState3} = parse_type(End, Module, ParseState2),
    {{range, BeginType, EndType}, ParseState3};
parse_type({type, _Line, union, Types}, Module, ParseState) ->
    {TypeList, ParseState2} = parse_type_list(Types, Module, ParseState),
    {{union, TypeList}, ParseState2};
parse_type({type, _Line, integer, []}, _Module, ParseState) ->
    {integer, ParseState};
parse_type({type, _Line, pos_integer, []}, _Module, ParseState) ->
    {pos_integer, ParseState};
parse_type({type, _Line, non_neg_integer, []}, _Module, ParseState) ->
    {non_neg_integer, ParseState};
parse_type({type, _Line, bool, []}, _Module, ParseState) ->
    {bool, ParseState};
parse_type({type, _Line, char, []}, _Module, ParseState) ->
    {{range, {integer, 0}, {integer, 16#10ffff}}, ParseState};
parse_type({type, _Line, string, []}, _Module, ParseState) ->
    {{list, {range, {integer, 0}, {integer, 16#10ffff}}}, ParseState};
parse_type({type, _Line, nonempty_string, []}, _Module, ParseState) ->
    {nonempty_string, ParseState};
parse_type({type, _Line, number, []}, _Module, ParseState) ->
    {{union, [integer, float]}, ParseState};
parse_type({type, _Line, boolean, []}, _Module, ParseState) ->
    {bool, ParseState};
parse_type({type, _Line, any, []}, _Module, ParseState) ->
    {{typedef, tester, test_any}, ParseState};
parse_type({type, _Line, atom, []}, _Module, ParseState) ->
    {atom, ParseState};
parse_type({type, _Line, binary, []}, _Module, ParseState) ->
    {binary, ParseState};
parse_type({type, _Line, pid, []}, _Module, ParseState) ->
    {pid, ParseState};
parse_type({type, _Line, port, []}, _Module, ParseState) ->
    {port, ParseState};
parse_type({type, _Line, float, []}, _Module, ParseState) ->
    {float, ParseState};
parse_type({type, _Line, iolist, []}, _Module, ParseState) ->
    {iolist, ParseState};
parse_type({type, _Line, nil, []}, _Module, ParseState) ->
    {nil, ParseState};
parse_type({type, _Line, node, []}, _Module, ParseState) ->
    {node, ParseState};
parse_type({type, _Line, none, []}, _Module, ParseState) ->
    {none, ParseState};
parse_type({type, _Line, reference, []}, _Module, ParseState) ->
    {reference, ParseState};
parse_type({type, _Line, term, []}, _Module, ParseState) ->
    {{typedef, tester, test_any}, ParseState};
parse_type({ann_type, _Line, [{var, _Line2, _Varname}, Type]}, Module, ParseState) ->
    parse_type(Type, Module, ParseState);
parse_type({atom, _Line, Atom}, _Module, ParseState) ->
    {{atom, Atom}, ParseState};<commit_msg>- support for builtin type tid() - print warning when type collector does not terminate, because of unknown types<commit_after>%  @copyright 2010 Konrad-Zuse-Zentrum fuer Informationstechnik Berlin
%  @end
%
%   Licensed under the Apache License, Version 2.0 (the "License");
%   you may not use this file except in compliance with the License.
%   You may obtain a copy of the License at
%
%       http://www.apache.org/licenses/LICENSE-2.0
%
%   Unless required by applicable law or agreed to in writing, software
%   distributed under the License is distributed on an "AS IS" BASIS,
%   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%   See the License for the specific language governing permissions and
%   limitations under the License.
%%%-------------------------------------------------------------------
%%% File    tester.erl
%%% @author <NAME> <<EMAIL>>
%%% @doc    test generator
%%% @end
%%% Created :  30 March 2010 by <NAME> <<EMAIL>>
%%%-------------------------------------------------------------------
%% @version $Id$
-module(tester).

-author('<EMAIL>').
-vsn('$Id$').

-export([test/4]).

% for unit tests
-export([create_value/3]).

-include_lib("tester.hrl").
-include_lib("unittest.hrl").

list_length_min() -> 1.
list_length_max() -> 5.

integer_min() -> 1.
integer_max() -> 5.

-spec test/4 :: (module(), atom(), non_neg_integer(), non_neg_integer()) -> ok.
test(Module, Func, Arity, Iterations) ->
    EmptyParseState = tester_parse_state:new_parse_state(),
    ParseState = collect_fun_info(Module, Func, Arity,
                                  EmptyParseState),
    run(Module, Func, Arity, Iterations, ParseState),
    ok.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% collect necessary type information
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-spec collect_fun_info/4 :: (module(), atom(), non_neg_integer(), #parse_state{}) -> #parse_state{}.
collect_fun_info(Module, Func, Arity, ParseState) ->
    ParseState2 = case tester_parse_state:lookup_type({'fun', Module, Func, Arity}, ParseState) of
                    {value, _} ->
                        ParseState;
                    none ->
                        {ok, {Module, [{abstract_code, {_AbstVersion, AbstractCode}}]}}
                            = beam_lib:chunks(code:where_is_file(atom_to_list(Module) ++ ".beam"), [abstract_code]),
                        lists:foldl(fun (Chunk, InnerParseState) ->
                                            parse_chunk(Chunk, Module, InnerParseState)
                                    end, ParseState, AbstractCode)
                end,
    ParseState3 = case tester_parse_state:get_unknown_types(ParseState2) of
        [] ->
            ParseState2;
        UnknownTypes ->
            collect_unknown_type_infos(UnknownTypes, ParseState2, 100)
    end,
    case tester_parse_state:lookup_type({'fun', Module, Func, Arity}, ParseState3) of
        {value, _} ->
            ParseState3;
        none ->
            ?ct_fail("unknown function ~p:~p/~p~n", [Module, Func, Arity])
    end.

-spec collect_unknown_type_infos/3 :: (list(type_name()), #parse_state{}, non_neg_integer()) -> #parse_state{}.
collect_unknown_type_infos(UnknownTypes, ParseState, 0) ->
    ct:pal("warning still looking for unknown types: ~p~n", [tester_parse_state:get_unknown_types(ParseState)]),
    collect_unknown_type_infos(UnknownTypes, ParseState, 100);
collect_unknown_type_infos(UnknownTypes, ParseState, Counter) ->
    ParseState2 = tester_parse_state:reset_unknown_types(ParseState),
    ParseState3 = lists:foldl(fun ({type, Module, TypeName}, InnerParseState) ->
                                        collect_type_info(Module, TypeName, InnerParseState)
                                end, ParseState2, UnknownTypes),
    case tester_parse_state:get_unknown_types(ParseState3) of
        [] ->
            ParseState3;
        NewUnknownTypes ->
            collect_unknown_type_infos(NewUnknownTypes, ParseState3, Counter - 1)
    end.

-spec collect_type_info/3 :: (module(), atom(), #parse_state{}) -> #parse_state{}.
collect_type_info(Module, Type, ParseState) ->
    case tester_parse_state:is_known_type(Module, Type, ParseState) of
        true ->
            ParseState;
        false ->
            {ok, {Module, [{abstract_code, {_AbstVersion, AbstractCode}}]}}
                = beam_lib:chunks(code:where_is_file(atom_to_list(Module) ++ ".beam"), [abstract_code]),
            lists:foldl(fun (Chunk, InnerParseState) ->
                                parse_chunk(Chunk, Module, InnerParseState)
                        end, ParseState, AbstractCode)
    end.


-spec parse_chunk/3 :: (any(), module(), #parse_state{}) -> #parse_state{}.
parse_chunk({attribute, _Line, type, {{record, TypeName}, ATypeSpec, _List}}, Module, ParseState) ->
    {TheTypeSpec, NewParseState} = parse_type(ATypeSpec, Module, ParseState),
    tester_parse_state:add_type_spec({type, Module, TypeName}, TheTypeSpec, NewParseState);
parse_chunk({attribute, _Line, type, {TypeName, ATypeSpec, _List}}, Module, ParseState) ->
    {TheTypeSpec, NewParseState} = parse_type(ATypeSpec, Module, ParseState),
    tester_parse_state:add_type_spec({type, Module, TypeName}, TheTypeSpec, NewParseState);
parse_chunk({attribute, _Line, 'spec', {{FunName, FunArity}, [AFunSpec]}}, Module, ParseState) ->
    {TheFunSpec, NewParseState} = parse_type(AFunSpec, Module, ParseState),
    tester_parse_state:add_type_spec({'fun', Module, FunName, FunArity}, TheFunSpec, NewParseState);
parse_chunk({attribute, _Line, record, {TypeName, TypeList}}, Module, ParseState) ->
    {TheTypeSpec, NewParseState} = parse_type(TypeList, Module, ParseState),
    tester_parse_state:add_type_spec({type, Module, TypeName}, TheTypeSpec, NewParseState);
parse_chunk({attribute, _Line, _AttributeName, _AttributeValue}, _Module, ParseState) ->
    ParseState;
parse_chunk({function, _Line, _FunName, _FunArity, FunCode}, _Module, ParseState) ->
    tester_value_collector:parse_expression(FunCode, ParseState);
parse_chunk({eof, _Line}, _Module, ParseState) ->
    ParseState.

-spec parse_type/3 :: (any(), module(), #parse_state{}) -> {type_spec() , #parse_state{}}.
parse_type({type, _Line, 'fun', [Arg, Result]}, Module, ParseState) ->
    {ArgType, ParseState2} = parse_type(Arg, Module, ParseState),
    {ResultType, ParseState3} = parse_type(Result, Module, ParseState2),
    {{'fun', ArgType, ResultType}, ParseState3};
parse_type({type, _Line, product, Types}, Module, ParseState) ->
    {TypeList, ParseState2} = parse_type_list(Types, Module, ParseState),
    {{product, TypeList}, ParseState2};
parse_type({type, _Line, tuple, any}, _Module, ParseState) ->
    {{tuple, {typedef, tester, test_any}}, ParseState};
parse_type({type, _Line, tuple, Types}, Module, ParseState) ->
    {TypeList, ParseState2} = parse_type_list(Types, Module, ParseState),
    {{tuple, TypeList}, ParseState2};
parse_type({type, _Line, list, [Type]}, Module, ParseState) ->
    {ListType, ParseState2} = parse_type(Type, Module, ParseState),
    {{list, ListType}, ParseState2};
parse_type({type, _Line, nonempty_list, [Type]}, Module, ParseState) ->
    {ListType, ParseState2} = parse_type(Type, Module, ParseState),
    {{nonempty_list, ListType}, ParseState2};
parse_type({type, _Line, list, []}, _Module, ParseState) ->
    {{list, {typedef, tester, test_any}}, ParseState};
parse_type({type, _Line, range, [Begin, End]}, Module, ParseState) ->
    {BeginType, ParseState2} = parse_type(Begin, Module, ParseState),
    {EndType, ParseState3} = parse_type(End, Module, ParseState2),
    {{range, BeginType, EndType}, ParseState3};
parse_type({type, _Line, union, Types}, Module, ParseState) ->
    {TypeList, ParseState2} = parse_type_list(Types, Module, ParseState),
    {{union, TypeList}, ParseState2};
parse_type({type, _Line, integer, []}, _Module, ParseState) ->
    {integer, ParseState};
parse_type({type, _Line, pos_integer, []}, _Module, ParseState) ->
    {pos_integer, ParseState};
parse_type({type, _Line, non_neg_integer, []}, _Module, ParseState) ->
    {non_neg_integer, ParseState};
parse_type({type, _Line, bool, []}, _Module, ParseState) ->
    {bool, ParseState};
parse_type({type, _Line, char, []}, _Module, ParseState) ->
    {{range, {integer, 0}, {integer, 16#10ffff}}, ParseState};
parse_type({type, _Line, string, []}, _Module, ParseState) ->
    {{list, {range, {integer, 0}, {integer, 16#10ffff}}}, ParseState};
parse_type({type, _Line, nonempty_string, []}, _Module, ParseState) ->
    {nonempty_string, ParseState};
parse_type({type, _Line, number, []}, _Module, ParseState) ->
    {{union, [integer, float]}, ParseState};
parse_type({type, _Line, boolean, []}, _Module, ParseState) ->
    {bool, ParseState};
parse_type({type, _Line, any, []}, _Module, ParseState) ->
    {{typedef, tester, test_any}, ParseState};
parse_type({type, _Line, atom, []}, _Module, ParseState) ->
    {atom, ParseState};
parse_type({type, _Line, binary, []}, _Module, ParseState) ->
    {binary, ParseState};
parse_type({type, _Line, pid, []}, _Module, ParseState) ->
    {pid, ParseState};
parse_type({type, _Line, tid, []}, _Module, ParseState) ->
    {tid, ParseState};
parse_type({type, _Line, port, []}, _Module, ParseState) ->
    {port, ParseState};
parse_type({type, _Line, float, []}, _Module, ParseState) ->
    {float, ParseState};
parse_type({type, _Line, iolist, []}, _Module, ParseState) ->
    {iolist, ParseState};
parse_type({type, _Line, nil, []}, _Module, ParseState) ->
    {nil, ParseState};
parse_type({type, _Line, node, []}, _Module, ParseState) ->
    {node, ParseState};
parse_type({type, _Line, none, []}, _Module, ParseState) ->
    {none, ParseState};
parse_type({type, _Line, reference, []}, _Module, ParseState) ->
    {reference, ParseState};
parse_type({type, _Line, term, []}, _Module, ParseState) ->
    {{typedef, tester, test_any}, ParseState};
parse_type({ann_type, _Line, [{var, _Line2, _Varname}, Type]}, Module, ParseState) ->
    parse_type(Type, Module, ParseState);
parse_type({atom, _Line, Atom}, _Module, ParseState) ->
    {{atom, Atom}, ParseState};
{'fraction_non_alphanumeric': 0.17665947518940345, 'fraction_numerical': 0.007224925994681651, 'mean_word_length': 3.87573385518591, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 151, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3549640', 'n_tokens_mistral': 6292, 'n_tokens_neox': 6123, 'n_words': 1444}