<commit_before>%% @author <NAME> <<EMAIL>>
%% @copyright 2010 <NAME>
%% @date 2010-03-31
%% @doc Store Zotonic rsc documents in Solr.
-module(solr_store).

-export([put/3, delete/3]).

-include("zotonic.hrl").


put(Id, Context, Solr) ->
    case convert(Id, Context) of
        undefined ->
            nop;
        Doc ->
            Doc1 = lists:filter(fun({_,undefined})->false; ({_,[]})->false; (_)->true end, Doc),
            %%?DEBUG(Doc1),
            esolr:add([{doc, Doc1}], Solr)
    end.


delete(Id, _Context, Solr) ->
    ok = esolr:delete({id, z_convert:to_list(Id)}, Solr).



%% See schema.xml to see which fields need to be put into Solr.
convert(Id, Context) ->
    All = m_rsc:get(Id, Context),

    StrVal = fun(Name) -> z_convert:to_list(first_trans(proplists:get_value(Name, All))) end,
    Bool = fun(Name) -> case proplists:get_value(Name, All) of
                            true -> "1";
                            false -> "0"
                        end
           end,
    Date = fun(Name) -> case proplists:get_value(Name, All) of
                            undefined -> undefined;
                            ?ST_JUTTEMIS -> "9999-08-17T12:00:00Z";
                            D -> to_isotime(D)
                        end
           end,

    %% Gather all text in the rsc, using the pivot routines.
    {_ObjIds, _CatIds, [TA,TB,TC,TD]} = z_pivot_rsc:get_pivot_data(Id, Context),
    AllText = lists:flatten(
                [
                 [ z_convert:to_list(X) || {_, X} <- TA ], " ",
                 [ z_convert:to_list(X) || {_, X} <- TB ], " ",
                 [ z_convert:to_list(X) || {_, X} <- TC ], " ",
                 [ z_convert:to_list(X) || {_, X} <- TD ]]),

    IsA = m_rsc:is_a(Id, Context),

    %% Module-based fields (starting with x_)
    ModuleProps = lists:flatten(z_notifier:foldl({solr_props, Id, IsA}, [], Context)),

    %% The returned document.
    []
        ++

        %% Regular rsc fields
        [{F, StrVal(F)} || F <- [id, version, uri, name, page_path, category_id, modifier_id, creator_id, visible_for]]<commit_msg>When end date is NULL in zotonic, set it to 9999 in solr.<commit_after>%% @author <NAME> <<EMAIL>>
%% @copyright 2010 <NAME>
%% @date 2010-03-31
%% @doc Store Zotonic rsc documents in Solr.
-module(solr_store).

-export([put/3, delete/3]).

-include("zotonic.hrl").


put(Id, Context, Solr) ->
    case convert(Id, Context) of
        undefined ->
            nop;
        Doc ->
            Doc1 = lists:filter(fun({_,undefined})->false; ({_,[]})->false; (_)->true end, Doc),
            %%?DEBUG(Doc1),
            esolr:add([{doc, Doc1}], Solr)
    end.


delete(Id, _Context, Solr) ->
    ok = esolr:delete({id, z_convert:to_list(Id)}, Solr).



%% See schema.xml to see which fields need to be put into Solr.
convert(Id, Context) ->
    All = m_rsc:get(Id, Context),

    StrVal = fun(Name) -> z_convert:to_list(first_trans(proplists:get_value(Name, All))) end,
    Bool = fun(Name) -> case proplists:get_value(Name, All) of
                            true -> "1";
                            false -> "0"
                        end
           end,
    Date = fun(Name) -> case proplists:get_value(Name, All) of
                            undefined -> "9999-08-17T12:00:00Z";
                            ?ST_JUTTEMIS -> "9999-08-17T12:00:00Z";
                            D -> to_isotime(D)
                        end
           end,

    %% Gather all text in the rsc, using the pivot routines.
    {_ObjIds, _CatIds, [TA,TB,TC,TD]} = z_pivot_rsc:get_pivot_data(Id, Context),
    AllText = lists:flatten(
                [
                 [ z_convert:to_list(X) || {_, X} <- TA ], " ",
                 [ z_convert:to_list(X) || {_, X} <- TB ], " ",
                 [ z_convert:to_list(X) || {_, X} <- TC ], " ",
                 [ z_convert:to_list(X) || {_, X} <- TD ]]),

    IsA = m_rsc:is_a(Id, Context),

    %% Module-based fields (starting with x_)
    ModuleProps = lists:flatten(z_notifier:foldl({solr_props, Id, IsA}, [], Context)),

    %% The returned document.
    []
        ++

        %% Regular rsc fields
        [{F, StrVal(F)} || F <- [id, version, uri, name, page_path, category_id, modifier_id, creator_id, visible_for]]
{'fraction_non_alphanumeric': 0.16223841993886667, 'fraction_numerical': 0.019750764166470727, 'mean_word_length': 1.9337931034482758, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1513913', 'n_tokens_mistral': 1547, 'n_tokens_neox': 1429, 'n_words': 323}