<commit_before>-module(ds_types).
-author("<NAME> <<EMAIL>>").

-export([ types/0
        , priv_data/1
        , priv_data/3
        ]).

-import(ds_opts, [ getopt/1
                 ]).

-include_lib("eunit/include/eunit.hrl").

%% The type hierarchy is defined by types()
%% - subtypes are mutually exclusive
%% - dynamically created subtypes denoted by
%%     {'$dynamic', Tag, Fun}
%%   where the actual subtype is derived as {Tag, Fun(Data)}
%%   (if Fun(Data) returns false, the next subtype is tried)
%% - dynamic subtypes must be last in the list of subtypes

types() ->
    [ {'T'
      , [ {numeric, fun erlang:is_number/1}
        , {atom, fun erlang:is_atom/1}
        , {list, fun erlang:is_list/1}
        , {tuple, fun erlang:is_tuple/1}
        ]}
    , {numeric
      , [ {integer, fun erlang:is_integer/1}
        , {float, fun erlang:is_float/1}
        ]}
    , {list
      , [ {empty, fun(L) -> L =:= [] end}
        , {str_printable, fun is_str_printable/1}
        , {'$dynamic', list, fun length/1}
        ]}
    , {str_printable
      , [ {str_alnum, fun is_str_alnum/1}
        ] ++ opt_strlen_stub(getopt(strlen)) }
    , {str_alnum
      , [ {str_integer, fun is_str_integer/1}
        , {str_alpha, fun is_str_alpha/1}
        ] ++ opt_strlen_stub(getopt(strlen)) }
    , {tuple
      , [ {empty, fun(T) -> T =:= {} end}
        , {'$dynamic', record, fun dyn_record/1}
        , {'$dynamic', tuple, fun size/1}
        ]}
    ] ++ opt_mag(getopt(mag))
      ++ opt_strlen(getopt(strlen)).

opt_mag(0) -> [];
opt_mag(N) -><commit_msg>ds_types: Add binary type<commit_after>-module(ds_types).
-author("<NAME> <<EMAIL>>").

-export([ types/0
        , priv_data/1
        , priv_data/3
        ]).

-import(ds_opts, [ getopt/1
                 ]).

-include_lib("eunit/include/eunit.hrl").

%% The type hierarchy is defined by types()
%% - subtypes are mutually exclusive
%% - dynamically created subtypes denoted by
%%     {'$dynamic', Tag, Fun}
%%   where the actual subtype is derived as {Tag, Fun(Data)}
%%   (if Fun(Data) returns false, the next subtype is tried)
%% - dynamic subtypes must be last in the list of subtypes

types() ->
    [ {'T'
      , [ {numeric, fun erlang:is_number/1}
        , {atom, fun erlang:is_atom/1}
        , {list, fun erlang:is_list/1}
        , {tuple, fun erlang:is_tuple/1}
        , {binary, fun erlang:is_binary/1}
        ]}
    , {numeric
      , [ {integer, fun erlang:is_integer/1}
        , {float, fun erlang:is_float/1}
        ]}
    , {list
      , [ {empty, fun(L) -> L =:= [] end}
        , {str_printable, fun is_str_printable/1}
        , {'$dynamic', list, fun length/1}
        ]}
    , {tuple
      , [ {empty, fun(T) -> T =:= {} end}
        , {'$dynamic', record, fun dyn_record/1}
        , {'$dynamic', tuple, fun size/1}
        ]}
    , {str_printable
      , [ {str_alnum, fun is_str_alnum/1}
        ] ++ opt_strlen_stub(getopt(strlen)) }
    , {str_alnum
      , [ {str_integer, fun is_str_integer/1}
        , {str_alpha, fun is_str_alpha/1}
        ] ++ opt_strlen_stub(getopt(strlen)) }
    ] ++ opt_mag(getopt(mag))
      ++ opt_strlen(getopt(strlen)).

opt_mag(0) -> [];
opt_mag(N) ->
{'fraction_non_alphanumeric': 0.1686028257456829, 'fraction_numerical': 0.011616954474097331, 'mean_word_length': 2.343126967471144, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '595439', 'n_tokens_mistral': 1163, 'n_tokens_neox': 1144, 'n_words': 246}