<commit_before>-module(dk_q).
-include_lib("proper/include/proper.hrl").

%% API
-export([execute/1]).

%% Record declarations ("q" is short for "query")
-record(and_q,  {l_sub_q :: q(), r_sub_q :: q()}).
-record(or_q,   {l_sub_q :: q(), r_sub_q :: q()}).
-record(not_q,  {sub     :: q()}).
-record(term_q, {keyword :: utf8_str()}).

%% Type definitions
-type q() :: {and_q,  q(), q()} |
             {or_q,   q(), q()} |
             {not_q,  q()     } |
             {term_q, utf8_str()}.
-type utf8_str() :: unicode:unicode_binary().

%%----------------------------------------------------------------------------
%% API
%%----------------------------------------------------------------------------

execute(Str) ->
    Cs = [partition(flatten(X)) || X <- and_subs(dnf(from_str(Str)))],
    Data = plists:mapreduce(
             fun fetch/1, 
             lists:usort(lists:flatten([Ts ++ Ns || {Ts,Ns} <- Cs]))),
    gb_sets:union(plists:map(fun (C) -> exec(C, Data) end, Cs)).

%%----------------------------------------------------------------------------
%% Internal functions
%%----------------------------------------------------------------------------

from_str(Str) ->
    {ok,ParseTree} = dk_q_parser:parse(scan(Str)),
    tree_to_query(ParseTree).

scan(<<C/utf8,Rest/bytes>>) ->
    case C of
        $( -> [{'(',1}|scan(Rest)];
        $) -> [{')',1}|scan(Rest)];
        $& -> [{'&',1}|scan(Rest)];
        $| -> [{'|',1}|scan(Rest)];<commit_msg>Add comments<commit_after>-module(dk_q).
-include_lib("proper/include/proper.hrl").

%% API
-export([execute/1]).

%% Record declarations ("q" is short for "query")
-record(and_q,  {l_sub_q :: q(), r_sub_q :: q()}).
-record(or_q,   {l_sub_q :: q(), r_sub_q :: q()}).
-record(not_q,  {sub     :: q()}).
-record(term_q, {keyword :: utf8_str()}).

%% Type definitions
-type q() :: {and_q,  q(), q()} |
             {or_q,   q(), q()} |
             {not_q,  q()     } |
             {term_q, utf8_str()}.
-type utf8_str() :: unicode:unicode_binary().

%%----------------------------------------------------------------------------
%% API
%%----------------------------------------------------------------------------

execute(Str) ->
    %% parse and preprocess query
    Cs = [partition(flatten(X)) || X <- and_subs(dnf(from_str(Str)))],
    %% fetch data
    Data = plists:mapreduce(
             fun fetch/1, 
             lists:usort(lists:flatten([Ts ++ Ns || {Ts,Ns} <- Cs]))),
    %% calculate result
    gb_sets:union(plists:map(fun (C) -> exec(C, Data) end, Cs)).

%%----------------------------------------------------------------------------
%% Internal functions
%%----------------------------------------------------------------------------

from_str(Str) ->
    {ok,ParseTree} = dk_q_parser:parse(scan(Str)),
    tree_to_query(ParseTree).

scan(<<C/utf8,Rest/bytes>>) ->
    case C of
        $( -> [{'(',1}|scan(Rest)];
        $) -> [{')',1}|scan(Rest)];
        $& -> [{'&',1}|scan(Rest)];
        $| -> [{'|',1}|scan(Rest)];
{'fraction_non_alphanumeric': 0.40293040293040294, 'fraction_numerical': 0.00666000666000666, 'mean_word_length': 1.4323886639676113, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1512224', 'n_tokens_mistral': 995, 'n_tokens_neox': 955, 'n_words': 164}