<commit_before>-module(dbpedia).

-export([
    search/1,
    describe/1,
    describe/2
]).

-include("zotonic.hrl").

%% @doc http://wiki.dbpedia.org/about/language-chapters
-define(SPARQL_ENDPOINT, <<"http://{lang}dbpedia.org/sparql">>).

search(#search_query{search = {dbpedia, Args}}) ->
    SparqlQuery = lists:foldl(
        fun({Key, Value}, Acc) ->
            parse_argument(Key, Value, Acc)
        end,
        <<"?s WHERE ">>,
        Args
    ),
    
    Language = z_convert:to_binary(proplists:get_value(lang, Args, <<>>)),
    Result = describe(SparqlQuery, Language),
    #search_result{result = [Result]}.

%% @doc Come up with alternative forms for RKD URIs, of which DBPedia has quite some.
parse_argument(rkd_uri, <<"https://rkd.nl/explore/artists/", Id/binary>> = Uri, Sparql) ->
    All = [
        Uri,
        <<"http://explore.rkd.nl/nl/explore/artists/", Id/binary>>
    ],
    Clauses = ["{?s <http://dbpedia.org/ontology/wikiPageExternalLink> <" ++ binary_to_list(U) ++ ">}"
        || U
        <- All
    ],
    Union = list_to_binary(string:join(Clauses, " UNION ")),
    <<Sparql/binary, "{", Union/binary, "}">>;
parse_argument(same_as, Uri, Sparql) ->
    <<Sparql/binary, "{?s <http://www.w3.org/2002/07/owl#sameAs> <", Uri/binary, "}">>;
parse_argument(_, _, Sparql) ->
    Sparql.

describe(Resource) when is_list(Resource) ->
    describe(list_to_binary(Resource));
describe(<<"http://", _/binary>> = Resource) ->
    describe(Resource, <<>>);
describe(Query) ->
    describe(Query, <<>>).

describe(Query, Language) when Language =:= <<"nl">>; Language =:= <<>> ->
    sparql_client:describe(endpoint(Language), Query);
describe(Resource, Language) when not is_list(Resource); not is_binary(Language) ->
    describe(z_convert:to_binary(Resource), z_convert:to_binary(Language)).

endpoint(<<>>) ->
    binary:replace(?SPARQL_ENDPOINT, <<"{lang}">>, <<>>);
endpoint(Language) ->
    binary:replace(?SPARQL_ENDPOINT, <<"{lang}">>, <<Language/binary, ".">>).

%%http://nl.dbpedia.org/sparql?default-graph-uri = http%3A%2F%2Fnl.dbpedia.org&query=select+distinct+%3Fconcept+where+%7B%3Fconcept+dbpedia-owl%3AwikiPageExternalLink+%3Chttp%3A%2F%2Fexplore.rkd.nl%2Fnl%2Fexplore%2Fartists%2F33671%3E%7D+LIMIT+100&format=text%2Fhtml&timeout=30000&debug=on
<commit_msg>[rdf] Add plain RKD id property to DBPedia SPARQL query for more matches<commit_after>-module(dbpedia).

-export([
    search/1,
    describe/1,
    describe/2
]).

-include("zotonic.hrl").

%% @doc http://wiki.dbpedia.org/about/language-chapters
-define(SPARQL_ENDPOINT, <<"http://{lang}dbpedia.org/sparql">>).

search(#search_query{search = {dbpedia, Args}}) ->
    SparqlQuery = lists:foldl(
        fun({Key, Value}, Acc) ->
            parse_argument(Key, Value, Acc)
        end,
        <<"?s WHERE ">>,
        Args
    ),
    
    Language = z_convert:to_binary(proplists:get_value(lang, Args, <<>>)),
    Result = describe(SparqlQuery, Language),
    #search_result{result = [Result]}.

%% @doc Come up with alternative forms for RKD URIs, of which DBPedia has quite some.
parse_argument(rkd_uri, <<"https://rkd.nl/explore/artists/", Id/binary>> = Uri, Sparql) ->
    Clause = [
        <<"{?s <http://dbpedia.org/ontology/wikiPageExternalLink> <", Uri/binary, ">}">>, <<" UNION ">>,
        %% alternative URI
        <<"{?s <http://dbpedia.org/ontology/wikiPageExternalLink> <http://explore.rkd.nl/nl/explore/artists/", Id/binary, ">}">>, <<" UNION ">>,
        %% alternative property
        <<"{?s <http://nl.dbpedia.org/property/rkd> ", Id/binary, "}">>
    ],
    <<Sparql/binary, "{", (iolist_to_binary(Clause))/binary, "}">>;
parse_argument(same_as, Uri, Sparql) ->
    <<Sparql/binary, "{?s <http://www.w3.org/2002/07/owl#sameAs> <", Uri/binary, "}">>;
parse_argument(_, _, Sparql) ->
    Sparql.

describe(Resource) when is_list(Resource) ->
    describe(list_to_binary(Resource));
describe(<<"http://", _/binary>> = Resource) ->
    describe(Resource, <<>>);
describe(Query) ->
    describe(Query, <<>>).

describe(Query, Language) when Language =:= <<"nl">>; Language =:= <<>> ->
    sparql_client:describe(endpoint(Language), Query);
describe(Resource, Language) when not is_list(Resource); not is_binary(Language) ->
    describe(z_convert:to_binary(Resource), z_convert:to_binary(Language)).

endpoint(<<>>) ->
    binary:replace(?SPARQL_ENDPOINT, <<"{lang}">>, <<>>);
endpoint(Language) ->
    binary:replace(?SPARQL_ENDPOINT, <<"{lang}">>, <<Language/binary, ".">>).

%%http://nl.dbpedia.org/sparql?default-graph-uri = http%3A%2F%2Fnl.dbpedia.org&query=select+distinct+%3Fconcept+where+%7B%3Fconcept+dbpedia-owl%3AwikiPageExternalLink+%3Chttp%3A%2F%2Fexplore.rkd.nl%2Fnl%2Fexplore%2Fartists%2F33671%3E%7D+LIMIT+100&format=text%2Fhtml&timeout=30000&debug=on

{'fraction_non_alphanumeric': 0.20501567398119122, 'fraction_numerical': 0.017136886102403343, 'mean_word_length': 4.300110741971207, 'pattern_counts': {'":': 0, '<': 91, '<?xml version=': 0, '>': 112, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5000569', 'n_tokens_mistral': 1870, 'n_tokens_neox': 1779, 'n_words': 244}