<commit_before>-module(ds).
-author("<NAME> <<EMAIL>>").

%% `dataspec' derives a spec of data based on a stream of values.
%% It can be used to eg. `discover' the data type stored in a table
%% (column, etc.)

-export([ add/2
        , new/0
        , new/1
        , sample_data/0
        , sample_data/1
        , sample_data/2
        , get_samples/1
        ]).

-include_lib("eunit/include/eunit.hrl").

%% Option handling: interface to ds_opts module
getopt(Opt) -> ds_opts:getopt(Opt).
setopts(Opts) -> ds_opts:setopts(Opts).

%% A data spec is a list of type specs.
%% It is most often hierarchically nested, hence we have
%% a hierarchical tree of type classes.
%% Nodes of this tree are represented as tuples:
%%
%% {Class, Count, SampleData, SubSpec | LeafData}
%%
%% - Class is a term (in most cases an atom) describing
%%   the type this node represents in the type hierarchy.
%%   Toplevel type examples: integer, atom, list.
%% - Count is an integer count of data items covered by
%%   this type class.
%% - SampleData contains samples of the data values and
%%   statistics derived from the data processed.
%% - SubSpec is a list of subtype nodes, if any, or [].
%%   It is up to the handler for Class to know if and how
%%   to further categorize a piece of data into subtypes.
%%   For leaf nodes, this holds type-specific leaf data,
%%   eg. for integers a distribution of values seen.

%% The type hierarchy is defined by types()
%% - subtypes are mutually exclusive
%% - dynamically created subtypes denoted by
%%     {'$dynamic', Tag, Fun}
%%   where the actual subtype is derived as {Tag, Fun(Data)}
%% - dynamic subtypes must be last in the list of subtypes

types() ->
    [ {root
      , [ {numeric, fun erlang:is_number/1}
        , {atom, fun erlang:is_atom/1}
        , {list, fun erlang:is_list/1}
        , {tuple, fun erlang:is_tuple/1}
        ]}
    , {numeric
      , [ {integer, fun erlang:is_integer/1}
        , {float, fun erlang:is_float/1}
        ]}
    , {list
      , [ {empty, fun(L) -> L =:= [] end}
        , {str_printable, fun is_str_printable/1}
        , {'$dynamic', list, fun length/1}
        ]}
    , {str_printable
      , [ {str_alnum, fun is_str_alnum/1}
        ]}
    , {str_alnum
      , [ {str_integer, fun is_str_integer/1}
        , {str_alpha, fun is_str_alpha/1}
        ]}
    , {tuple
      , [ {empty, fun(T) -> T =:= {} end}
        , {'$dynamic', tuple, fun size/1}
        ]}
    ] ++ optional_types().

optional_types() ->
    case getopt(mag) of
        0 -> [];
        N -> [ {integer, [ {'$dynamic', mag, fun(V) -> mag(V, N) end} ]}
             , {float,   [ {'$dynamic', mag, fun(V) -> mag(V, N) end} ]}
             ]
    end.

%% Initialize a new data spec
new() -> new(root).

new(Class) -> {Class, 0, sample_data(), []}.

%% Initialize, immediately adding one data term
new(Class, Data) -> add(Data, new(Class)).

%% Add an instance of Data to Spec.
%% If Spec already contains Data, do nothing.
%% Else, extend Spec in the smallest possible way to include Data.
%% add/2 is written so it can be used as a function to lists:foldl/3.
%%
%% When adding a value to the spec:
%% - each class knows which subtype the data fits in.
%% Adding a new value is a recursive process:
%% - starting from the root class, each class
%%   - accounts for the new value itself;
%%   - chooses appropriate subtype (if any) and
%%   - passes the value to that subtype.
%%
%% Eg. when adding the value 100, 'root' accounts
%% for this (increases counter of values, etc),
%% determines that it is a numeric type, and so
%% passes it to 'numeric' which, in turn, determines
%% it is an integer and passes to 'integer', which
%% might still subtype it based on its magnitude.
%% On all levels of the hierarchy, counters will be
%% increased and samples will be collected.

add(V, {Class, Count, SD, SubSpec}) ->
    case lists:keyfind(Class, 1, types()) of
        false -> %% leaf type
            {Class, Count+1, sample_data(V, SD), leaf_data(V, Class, SubSpec)};
        {Class, SubTypes} -> %% abstract type
            SubType = subtype(V, SubTypes),
            {Class, Count+1, sample_data(V, SD), merge(V, SubType, SubSpec)}
    end.

%% choose the appropriate subtype based on the filters
%% in the type hierarchy, or dynamically generate subtype.
subtype(_V, []) -> untyped;
subtype(V, [{'$dynamic', SubTag, SubFun} | _Rest]) ->
    {SubTag, SubFun(V)};
subtype(V, [{SubType, FilterFun} | Rest]) ->
    case FilterFun(V) of
        true  -> SubType;
        false -> subtype(V, Rest)
    end.

%% choose subspec given by Class or create it from scratch,
%% add V to it and returns the resulting Spec.
merge(V, Class, Spec) ->
    case lists:keyfind(Class, 1, Spec) of
        false   -> [new(Class, V) | Spec];
        SubSpec -> lists:keystore(Class, 1, Spec, add(V, SubSpec))
    end.

leaf_data(V, atom, LeafData) ->
    orddict:update_counter(V, 1, LeafData);
leaf_data(V, {list, _N}, LeafData) ->
    leaf_data_recur(V, LeafData);
leaf_data(V, {tuple, _N}, LeafData) ->
    leaf_data_recur(tuple_to_list(V), LeafData);
%% Add clauses for various Class to extend LeafData with V
%% eg. min, max, distrib, zero-count etc. for integers.
%% The interpretation of this data is class-specific.
leaf_data(_V, _Class, LeafData) -> LeafData.

%% For lists and tuples, the LeafData stores a list of type specs
%% for each list/tuple element position.
leaf_data_recur(V, []) ->
    lists:map(fun(D) -> new(root, D) end, V);
leaf_data_recur(V, LeafData) -><commit_msg>Rename root type from 'root' to 'T'<commit_after>-module(ds).
-author("<NAME> <<EMAIL>>").

%% `dataspec' derives a spec of data based on a stream of values.
%% It can be used to eg. `discover' the data type stored in a table
%% (column, etc.)

-export([ add/2
        , new/0
        , new/1
        , sample_data/0
        , sample_data/1
        , sample_data/2
        , get_samples/1
        ]).

-include_lib("eunit/include/eunit.hrl").

%% Option handling: interface to ds_opts module
getopt(Opt) -> ds_opts:getopt(Opt).
setopts(Opts) -> ds_opts:setopts(Opts).

%% A data spec is a list of type specs.
%% It is most often hierarchically nested, hence we have
%% a hierarchical tree of type classes.
%% Nodes of this tree are represented as tuples:
%%
%% {Class, Count, SampleData, SubSpec | LeafData}
%%
%% - Class is a term (in most cases an atom) describing
%%   the type this node represents in the type hierarchy.
%%   Toplevel type examples: integer, atom, list.
%% - Count is an integer count of data items covered by
%%   this type class.
%% - SampleData contains samples of the data values and
%%   statistics derived from the data processed.
%% - SubSpec is a list of subtype nodes, if any, or [].
%%   It is up to the handler for Class to know if and how
%%   to further categorize a piece of data into subtypes.
%%   For leaf nodes, this holds type-specific leaf data,
%%   eg. for integers a distribution of values seen.

%% The type hierarchy is defined by types()
%% - subtypes are mutually exclusive
%% - dynamically created subtypes denoted by
%%     {'$dynamic', Tag, Fun}
%%   where the actual subtype is derived as {Tag, Fun(Data)}
%% - dynamic subtypes must be last in the list of subtypes

types() ->
    [ {'T'
      , [ {numeric, fun erlang:is_number/1}
        , {atom, fun erlang:is_atom/1}
        , {list, fun erlang:is_list/1}
        , {tuple, fun erlang:is_tuple/1}
        ]}
    , {numeric
      , [ {integer, fun erlang:is_integer/1}
        , {float, fun erlang:is_float/1}
        ]}
    , {list
      , [ {empty, fun(L) -> L =:= [] end}
        , {str_printable, fun is_str_printable/1}
        , {'$dynamic', list, fun length/1}
        ]}
    , {str_printable
      , [ {str_alnum, fun is_str_alnum/1}
        ]}
    , {str_alnum
      , [ {str_integer, fun is_str_integer/1}
        , {str_alpha, fun is_str_alpha/1}
        ]}
    , {tuple
      , [ {empty, fun(T) -> T =:= {} end}
        , {'$dynamic', tuple, fun size/1}
        ]}
    ] ++ optional_types().

optional_types() ->
    case getopt(mag) of
        0 -> [];
        N -> [ {integer, [ {'$dynamic', mag, fun(V) -> mag(V, N) end} ]}
             , {float,   [ {'$dynamic', mag, fun(V) -> mag(V, N) end} ]}
             ]
    end.

%% Initialize a new data spec
new() -> new('T').

new(Class) -> {Class, 0, sample_data(), []}.

%% Initialize, immediately adding one data term
new(Class, Data) -> add(Data, new(Class)).

%% Add an instance of Data to Spec.
%% If Spec already contains Data, do nothing.
%% Else, extend Spec in the smallest possible way to include Data.
%% add/2 is written so it can be used as a function to lists:foldl/3.
%%
%% When adding a value to the spec:
%% - each class knows which subtype the data fits in.
%% Adding a new value is a recursive process:
%% - starting from 'T' (root type) class, each class
%%   - accounts for the new value itself;
%%   - chooses appropriate subtype (if any) and
%%   - passes the value to that subtype.
%%
%% Eg. when adding the value 100, 'T' accounts
%% for this (increases counter of values, etc),
%% determines that it is a numeric type, and so
%% passes it to 'numeric' which, in turn, determines
%% it is an integer and passes to 'integer', which
%% might still subtype it based on its magnitude.
%% On all levels of the hierarchy, counters will be
%% increased and samples will be collected.

add(V, {Class, Count, SD, SubSpec}) ->
    case lists:keyfind(Class, 1, types()) of
        false -> %% leaf type
            {Class, Count+1, sample_data(V, SD), leaf_data(V, Class, SubSpec)};
        {Class, SubTypes} -> %% abstract type
            SubType = subtype(V, SubTypes),
            {Class, Count+1, sample_data(V, SD), merge(V, SubType, SubSpec)}
    end.

%% choose the appropriate subtype based on the filters
%% in the type hierarchy, or dynamically generate subtype.
subtype(_V, []) -> untyped;
subtype(V, [{'$dynamic', SubTag, SubFun} | _Rest]) ->
    {SubTag, SubFun(V)};
subtype(V, [{SubType, FilterFun} | Rest]) ->
    case FilterFun(V) of
        true  -> SubType;
        false -> subtype(V, Rest)
    end.

%% choose subspec given by Class or create it from scratch,
%% add V to it and returns the resulting Spec.
merge(V, Class, Spec) ->
    case lists:keyfind(Class, 1, Spec) of
        false   -> [new(Class, V) | Spec];
        SubSpec -> lists:keystore(Class, 1, Spec, add(V, SubSpec))
    end.

leaf_data(V, atom, LeafData) ->
    orddict:update_counter(V, 1, LeafData);
leaf_data(V, {list, _N}, LeafData) ->
    leaf_data_recur(V, LeafData);
leaf_data(V, {tuple, _N}, LeafData) ->
    leaf_data_recur(tuple_to_list(V), LeafData);
%% Add clauses for various Class to extend LeafData with V
%% eg. min, max, distrib, zero-count etc. for integers.
%% The interpretation of this data is class-specific.
leaf_data(_V, _Class, LeafData) -> LeafData.

%% For lists and tuples, the LeafData stores a list of type specs
%% for each list/tuple element position.
leaf_data_recur(V, []) ->
    lists:map(fun(D) -> new('T', D) end, V);
leaf_data_recur(V, LeafData) ->
{'fraction_non_alphanumeric': 0.1360151528817534, 'fraction_numerical': 0.005772526382249481, 'mean_word_length': 2.9459074733096084, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5223686', 'n_tokens_mistral': 3640, 'n_tokens_neox': 3507, 'n_words': 1317}