<commit_before>-module(library).
-behaviour(gen_server).
-behaviour(playdar_resolver).
-include_lib("stdlib/include/qlc.hrl").

%% API
-export([start_link/0, resolve/3, weight/1, targettime/1, name/1]).
-export([scan/2, generate_index/1 ]).

%% gen_server callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).

%-export([ngram/1]).
%-export([ngram/2]).
%-export([list_agg/1, list_add/1]).

%% Records
-record(state, {scanner}).
-record(file, {url,
               artist, album, track, 
               artist_clean, album_clean, track_clean,
               size, mtime, hash, mimetype, duration, bitrate, trackno}).
-record(ngram, {gram, name, num}).

%%

start_link()        -> gen_server:start_link(?MODULE, [], []).
scan(Pid, Dir)      -> gen_server:cast(Pid, {scan, Dir}).
generate_index(Pid) -> gen_server:cast(Pid, {generate_index}).

resolve(Pid, Q, Qpid)   -> gen_server:cast(Pid, {resolve, Q, Qpid}).
weight(_Pid)            -> 100.
targettime(_Pid)        -> 20.
name(_Pid)              -> "Local Library".

%%

init([]) ->
    ok = mnesia:start(),
    mnesia:wait_for_tables([file, ngram], 10000),
    try
        Info = mnesia:table_info(file, all),
        io:format("~w\n",[Info])
    catch
        exit:_Why -> % because it doesnt exist yet?
            first_run();
        X ->
            io:format("Mnesia table_info failed somehow.\n",[]),
            throw(X)
    end,    
    resolver:add_resolver(?MODULE, name(self()), weight(self()), targettime(self()), self()),
    {ok, #state{scanner=undefined}}.



handle_cast({resolve, Q, Qpid}, State) ->
    case Q of
        {struct, Mq} -> % Mq is a proplist
            Report = fun(Score, File) ->
                Rep =   {struct, [
                                {<<"artist">>, File#file.artist},
                                {<<"track">>,  File#file.track},
                                {<<"album">>,  File#file.album},
                                {<<"mimetype">>, File#file.mimetype},
                                {<<"score">>, Score},
                                {<<"url">>, File#file.url}
                                
                            ]},
                qry:add_result(Qpid, Rep)                
            end,
            Art = proplists:get_value(<<"artist">>,Mq,<<"">>),
            Trk = proplists:get_value(<<"track">>,Mq,<<"">>),
            Alb = proplists:get_value(<<"album">>,Mq,<<"">>),
            Files = search(clean(Art),clean(Alb),clean(Trk)),
            [ Report(Score, File) || {Score, File} <- Files ];

        _ -> io:format("Unhandled query type in library resolver~n",[])
    end,
    {noreply, State};

handle_cast({generate_index}, State) ->
    {atomic, L} = mnesia:transaction(fun()-> qlc:e(qlc:q(
                       [ {string:to_lower(binary_to_list(F#file.artist)), 
                          string:to_lower(binary_to_list(F#file.track))}
                         || F <- mnesia:table(file) 
                       ] )) end),
    Larts = [N || {N,_} <- lists:ukeysort(1, L)],
    Ltrks = [N || {_,N} <- lists:ukeysort(2, L)],
    Fun = fun(Name, Type)->
                        Ns = [ #ngram{gram={Type, Gram}, name=Name, num=Num} 
                               || {Gram, Num} <- ngram(Name)],
                        lists:foreach(fun(E)->mnesia:dirty_write(E)end, Ns)
          end,
    lists:foreach(fun(E)->Fun(E,artist)end, Larts),
    lists:foreach(fun(E)->Fun(E,track)end, Ltrks),<commit_msg>report bitrate, duration etc<commit_after>-module(library).
-behaviour(gen_server).
-behaviour(playdar_resolver).
-include_lib("stdlib/include/qlc.hrl").

%% API
-export([start_link/0, resolve/3, weight/1, targettime/1, name/1]).
-export([scan/2, generate_index/1 ]).

%% gen_server callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).

%-export([ngram/1]).
%-export([ngram/2]).
%-export([list_agg/1, list_add/1]).

%% Records
-record(state, {scanner}).
-record(file, {url,
               artist, album, track, 
               artist_clean, album_clean, track_clean,
               size, mtime, hash, mimetype, duration, bitrate, trackno}).
-record(ngram, {gram, name, num}).

%%

start_link()        -> gen_server:start_link(?MODULE, [], []).
scan(Pid, Dir)      -> gen_server:cast(Pid, {scan, Dir}).
generate_index(Pid) -> gen_server:cast(Pid, {generate_index}).

resolve(Pid, Q, Qpid)   -> gen_server:cast(Pid, {resolve, Q, Qpid}).
weight(_Pid)            -> 100.
targettime(_Pid)        -> 20.
name(_Pid)              -> "Local Library".

%%

init([]) ->
    ok = mnesia:start(),
    mnesia:wait_for_tables([file, ngram], 10000),
    try
        Info = mnesia:table_info(file, all),
        io:format("~w\n",[Info])
    catch
        exit:_Why -> % because it doesnt exist yet?
            first_run();
        X ->
            io:format("Mnesia table_info failed somehow.\n",[]),
            throw(X)
    end,    
    resolver:add_resolver(?MODULE, name(self()), weight(self()), targettime(self()), self()),
    {ok, #state{scanner=undefined}}.



handle_cast({resolve, Q, Qpid}, State) ->
    case Q of
        {struct, Mq} -> % Mq is a proplist
            Report = fun(Score, File) ->
                Rep =   {struct, [
                                {<<"artist">>, File#file.artist},
                                {<<"track">>,  File#file.track},
                                {<<"album">>,  File#file.album},
                                {<<"mimetype">>, File#file.mimetype},
                                {<<"score">>, Score},
                                {<<"url">>, File#file.url},
                                {<<"duration">>, File#file.duration},
                                {<<"bitrate">>, File#file.bitrate},
                                {<<"size">>, File#file.size}
                            ]},
                qry:add_result(Qpid, Rep)                
            end,
            Art = proplists:get_value(<<"artist">>,Mq,<<"">>),
            Trk = proplists:get_value(<<"track">>,Mq,<<"">>),
            Alb = proplists:get_value(<<"album">>,Mq,<<"">>),
            Files = search(clean(Art),clean(Alb),clean(Trk)),
            [ Report(Score, File) || {Score, File} <- Files ];

        _ -> io:format("Unhandled query type in library resolver~n",[])
    end,
    {noreply, State};

handle_cast({generate_index}, State) ->
    {atomic, L} = mnesia:transaction(fun()-> qlc:e(qlc:q(
                       [ {string:to_lower(binary_to_list(F#file.artist)), 
                          string:to_lower(binary_to_list(F#file.track))}
                         || F <- mnesia:table(file) 
                       ] )) end),
    Larts = [N || {N,_} <- lists:ukeysort(1, L)],
    Ltrks = [N || {_,N} <- lists:ukeysort(2, L)],
    Fun = fun(Name, Type)->
                        Ns = [ #ngram{gram={Type, Gram}, name=Name, num=Num} 
                               || {Gram, Num} <- ngram(Name)],
                        lists:foreach(fun(E)->mnesia:dirty_write(E)end, Ns)
          end,
    lists:foreach(fun(E)->Fun(E,artist)end, Larts),
    lists:foreach(fun(E)->Fun(E,track)end, Ltrks),
{'fraction_non_alphanumeric': 0.18512834821428573, 'fraction_numerical': 0.008091517857142858, 'mean_word_length': 1.9502057613168724, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 97, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1446921', 'n_tokens_mistral': 2467, 'n_tokens_neox': 2357, 'n_words': 402}