<commit_before>/** \file   FullTextCache.cc
 *  \brief  Implementation of functions relating to our full-text cache.
 *  \author Dr. <NAME> (<EMAIL>)
 *
 *  \copyright 2017-2020 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "FullTextCache.h"
#include <algorithm>
#include <sstream>
#include <tuple>
#include <ctime>
#include "Compiler.h"
#include "DbRow.h"
#include "FileUtil.h"
#include "PdfUtil.h"
#include "Random.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "TimeUtil.h"
#include "UrlUtil.h"
#include "VuFind.h"
#include "util.h"


constexpr unsigned MIN_CACHE_EXPIRE_TIME_ON_ERROR(42300 * 60);     // About 1 month in seconds.
constexpr unsigned MAX_CACHE_EXPIRE_TIME_ON_ERROR(42300 * 60 * 2); // About 2 months in seconds.


static const std::map<std::string, FullTextCache::TextType> description_to_text_type_map{ { "Volltext", FullTextCache::FULLTEXT },
                                                                                          { "Inhaltsverzeichnis", FullTextCache::TOC },
                                                                                          { "Inhaltstext", FullTextCache::ABSTRACT },
                                                                                          { "Zusammenfassung", FullTextCache::SUMMARY },
                                                                                          { "Literaturverzeichnis",
                                                                                            FullTextCache::LIST_OF_REFERENCES } };


bool FullTextCache::getDomainFromUrl(const std::string &url, std::string * const domain) const {
    std::string scheme, username_password, authority, port, path, params, query, fragment, relative_url;
    const bool result(
        UrlUtil::ParseUrl(url, &scheme, &username_password, &authority, &port, &path, &params, &query, &fragment, &relative_url));
    if (result)
        *domain = authority;

    return result;
}


bool FullTextCache::entryExpired(const std::string &id, std::vector<std::string> urls) {
    Entry entry;
    if (not getEntry(id, &entry))
        return true;
    const time_t now(std::time(nullptr));
    if (entry.expiration_ == TimeUtil::BAD_TIME_T or now < entry.expiration_) {
        std::vector<std::string> existing_urls(FullTextCache::getEntryUrlsAsStrings(id));

        std::sort(existing_urls.begin(), existing_urls.end());
        std::sort(urls.begin(), urls.end());
        if (urls == existing_urls)
            return false;
    }

    return true;
}


bool FullTextCache::singleUrlExpired(const std::string &id, const std::string &url) {
    Entry entry;
    if (not getEntry(id, &entry))
        return true;
    if (entry.expiration_ == TimeUtil::BAD_TIME_T or (std::time(nullptr) < entry.expiration_)) {
        for (const auto &entry_url : FullTextCache::getEntryUrls(id)) {
            if (entry_url.url_ == url and entry_url.error_message_.empty())
                return false;
        }
    }
    return true;
}


bool FullTextCache::dummyEntryExists(const std::string &id) {
    Entry entry;
    if (not getEntry(id, &entry))
        return false;
    for (const auto &entry_url : FullTextCache::getEntryUrls(id)) {
        if (entry_url.url_ == DUMMY_URL)
            return true;
    }
    return false;
}


void FullTextCache::expireEntries() {
    full_text_cache_urls_.deleteRange("expiration", Elasticsearch::RO_LTE, "now");
}


bool FullTextCache::getEntry(const std::string &id, Entry * const entry) const {
    const auto results(full_text_cache_.simpleSelect({ "expiration" }, "id", id));
    if (results.empty())
        return false;

    entry->id_ = id;
    if (results.front().find("expiration") == results.front().cend())
        entry->expiration_ = TimeUtil::BAD_TIME_T;
    else
        entry->expiration_ = TimeUtil::Iso8601StringToTimeT(results.front().find("expiration")->second);
    return true;
}


bool FullTextCache::hasEntryWithType(const std::string &id, const TextType &text_type) const {
    Entry entry;
    if (not getEntry(id, &entry))
        return false;
    const auto results(full_text_cache_.simpleSelect({ "text_type" }, "id", id));
    for (const auto &result : results) {
        const auto result_type(result.find("text_type"));
        if (result_type == result.cend())
            continue;
        if (result_type->second == std::to_string(text_type))
            return true;
    }
    return false;
}


inline std::string GetValueOrEmptyString(const std::map<std::string, std::string> &map, const std::string &key) {
    const auto pair(map.find(key));
    return pair == map.cend() ? "" : pair->second;
}


std::vector<FullTextCache::EntryUrl> FullTextCache::getEntryUrls(const std::string &id) const {
    const auto results(full_text_cache_urls_.simpleSelect({ "url", "domain", "error_message" }, "id", id));

    std::vector<EntryUrl> entry_urls;
    entry_urls.reserve(results.size());
    for (const auto &map : results) {
        EntryUrl entry_url;
        entry_url.id_ = id;
        entry_url.url_ = GetValueOrEmptyString(map, "url");
        entry_url.domain_ = GetValueOrEmptyString(map, "domain");
        entry_url.error_message_ = GetValueOrEmptyString(map, "error_message");
        entry_urls.emplace_back(entry_url);
    }
    return entry_urls;
}


std::vector<std::string> FullTextCache::getEntryUrlsAsStrings(const std::string &id) const {
    const auto results(full_text_cache_urls_.simpleSelect({ "url" }, "id", id));

    std::vector<std::string> urls;
    for (const auto &map : results) {
        const auto url(GetValueOrEmptyString(map, "url"));
        if (not url.empty())
            urls.emplace_back(url);
    }
    return urls;
}


unsigned FullTextCache::getErrorCount() const {
    return full_text_cache_urls_.count({ { "(error_message", "[ * TO * ]) AND -(error_message:DUMMY\\ ERROR)" } });
}


bool FullTextCache::getFullText(const std::string &id, std::string * const full_text) const {
    const auto results(full_text_cache_.simpleSelect({ "full_text" }, "id", id));

    if (results.empty())
        return false;

    *full_text = GetValueOrEmptyString(results.front(), "full_text");
    return true;
}


const std::string US("\x1F"); // ASCII unit separator


std::vector<FullTextCache::EntryGroup> FullTextCache::getEntryGroupsByDomainAndErrorMessage() const {
    const auto results(full_text_cache_urls_.simpleSelect({ "url", "domain", "error_message", "id" }));

    std::unordered_map<std::string, std::tuple<std::string, std::string, unsigned>> domains_and_errors_to_ids_and_urls_and_counts_map;
    for (const auto &map : results) {
        const auto url_pair(map.find("url"));
        const auto domain_pair(map.find("domain"));
        const auto error_message_pair(map.find("error_message"));
        if (url_pair == map.cend() or domain_pair == map.cend() or error_message_pair == map.cend())
            continue;

        const auto id_pair(map.find("id"));
        const auto key(domain_pair->second + US + error_message_pair->second);
        auto id_and_url_and_count(domains_and_errors_to_ids_and_urls_and_counts_map.find(key));
        if (id_and_url_and_count != domains_and_errors_to_ids_and_urls_and_counts_map.end())<commit_msg>Fix error extraction<commit_after>/** \file   FullTextCache.cc
 *  \brief  Implementation of functions relating to our full-text cache.
 *  \author Dr. <NAME> (<EMAIL>)
 *
 *  \copyright 2017-2020 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "FullTextCache.h"
#include <algorithm>
#include <sstream>
#include <tuple>
#include <ctime>
#include "Compiler.h"
#include "DbRow.h"
#include "FileUtil.h"
#include "PdfUtil.h"
#include "Random.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "TimeUtil.h"
#include "UrlUtil.h"
#include "VuFind.h"
#include "util.h"


constexpr unsigned MIN_CACHE_EXPIRE_TIME_ON_ERROR(42300 * 60);     // About 1 month in seconds.
constexpr unsigned MAX_CACHE_EXPIRE_TIME_ON_ERROR(42300 * 60 * 2); // About 2 months in seconds.


static const std::map<std::string, FullTextCache::TextType> description_to_text_type_map{ { "Volltext", FullTextCache::FULLTEXT },
                                                                                          { "Inhaltsverzeichnis", FullTextCache::TOC },
                                                                                          { "Inhaltstext", FullTextCache::ABSTRACT },
                                                                                          { "Zusammenfassung", FullTextCache::SUMMARY },
                                                                                          { "Literaturverzeichnis",
                                                                                            FullTextCache::LIST_OF_REFERENCES } };


bool FullTextCache::getDomainFromUrl(const std::string &url, std::string * const domain) const {
    std::string scheme, username_password, authority, port, path, params, query, fragment, relative_url;
    const bool result(
        UrlUtil::ParseUrl(url, &scheme, &username_password, &authority, &port, &path, &params, &query, &fragment, &relative_url));
    if (result)
        *domain = authority;

    return result;
}


bool FullTextCache::entryExpired(const std::string &id, std::vector<std::string> urls) {
    Entry entry;
    if (not getEntry(id, &entry))
        return true;
    const time_t now(std::time(nullptr));
    if (entry.expiration_ == TimeUtil::BAD_TIME_T or now < entry.expiration_) {
        std::vector<std::string> existing_urls(FullTextCache::getEntryUrlsAsStrings(id));

        std::sort(existing_urls.begin(), existing_urls.end());
        std::sort(urls.begin(), urls.end());
        if (urls == existing_urls)
            return false;
    }

    return true;
}


bool FullTextCache::singleUrlExpired(const std::string &id, const std::string &url) {
    Entry entry;
    if (not getEntry(id, &entry))
        return true;
    if (entry.expiration_ == TimeUtil::BAD_TIME_T or (std::time(nullptr) < entry.expiration_)) {
        for (const auto &entry_url : FullTextCache::getEntryUrls(id)) {
            if (entry_url.url_ == url and entry_url.error_message_.empty())
                return false;
        }
    }
    return true;
}


bool FullTextCache::dummyEntryExists(const std::string &id) {
    Entry entry;
    if (not getEntry(id, &entry))
        return false;
    for (const auto &entry_url : FullTextCache::getEntryUrls(id)) {
        if (entry_url.url_ == DUMMY_URL)
            return true;
    }
    return false;
}


void FullTextCache::expireEntries() {
    full_text_cache_urls_.deleteRange("expiration", Elasticsearch::RO_LTE, "now");
}


bool FullTextCache::getEntry(const std::string &id, Entry * const entry) const {
    const auto results(full_text_cache_.simpleSelect({ "expiration" }, "id", id));
    if (results.empty())
        return false;

    entry->id_ = id;
    if (results.front().find("expiration") == results.front().cend())
        entry->expiration_ = TimeUtil::BAD_TIME_T;
    else
        entry->expiration_ = TimeUtil::Iso8601StringToTimeT(results.front().find("expiration")->second);
    return true;
}


bool FullTextCache::hasEntryWithType(const std::string &id, const TextType &text_type) const {
    Entry entry;
    if (not getEntry(id, &entry))
        return false;
    const auto results(full_text_cache_.simpleSelect({ "text_type" }, "id", id));
    for (const auto &result : results) {
        const auto result_type(result.find("text_type"));
        if (result_type == result.cend())
            continue;
        if (result_type->second == std::to_string(text_type))
            return true;
    }
    return false;
}


inline std::string GetValueOrEmptyString(const std::map<std::string, std::string> &map, const std::string &key) {
    const auto pair(map.find(key));
    return pair == map.cend() ? "" : pair->second;
}


std::vector<FullTextCache::EntryUrl> FullTextCache::getEntryUrls(const std::string &id) const {
    const auto results(full_text_cache_urls_.simpleSelect({ "url", "domain", "error_message" }, "id", id));

    std::vector<EntryUrl> entry_urls;
    entry_urls.reserve(results.size());
    for (const auto &map : results) {
        EntryUrl entry_url;
        entry_url.id_ = id;
        entry_url.url_ = GetValueOrEmptyString(map, "url");
        entry_url.domain_ = GetValueOrEmptyString(map, "domain");
        entry_url.error_message_ = GetValueOrEmptyString(map, "error_message");
        entry_urls.emplace_back(entry_url);
    }
    return entry_urls;
}


std::vector<std::string> FullTextCache::getEntryUrlsAsStrings(const std::string &id) const {
    const auto results(full_text_cache_urls_.simpleSelect({ "url" }, "id", id));

    std::vector<std::string> urls;
    for (const auto &map : results) {
        const auto url(GetValueOrEmptyString(map, "url"));
        if (not url.empty())
            urls.emplace_back(url);
    }
    return urls;
}


unsigned FullTextCache::getErrorCount() const {
    return full_text_cache_urls_.count({ { "(error_message", "[ * TO * ]) AND -(error_message:DUMMY\\ ERROR)" } });
}


bool FullTextCache::getFullText(const std::string &id, std::string * const full_text) const {
    const auto results(full_text_cache_.simpleSelect({ "full_text" }, "id", id));

    if (results.empty())
        return false;

    *full_text = GetValueOrEmptyString(results.front(), "full_text");
    return true;
}


const std::string US("\x1F"); // ASCII unit separator


std::vector<FullTextCache::EntryGroup> FullTextCache::getEntryGroupsByDomainAndErrorMessage() const {
    const auto results(full_text_cache_urls_.simpleSelect({ "url", "domain", "error_message", "id" }, {}, UINT_MAX));

    std::unordered_map<std::string, std::tuple<std::string, std::string, unsigned>> domains_and_errors_to_ids_and_urls_and_counts_map;
    for (const auto &map : results) {
        const auto url_pair(map.find("url"));
        const auto domain_pair(map.find("domain"));
        const auto error_message_pair(map.find("error_message"));
        if (url_pair == map.cend() or domain_pair == map.cend() or error_message_pair == map.cend()
            or error_message_pair->second == FullTextCache::DUMMY_ERROR)
                continue;

        const auto id_pair(map.find("id"));
        const auto key(domain_pair->second + US + error_message_pair->second);
        auto id_and_url_and_count(domains_and_errors_to_ids_and_urls_and_counts_map.find(key));
        if (id_and_url_and_count != domains_and_errors_to_ids_and_urls_and_counts_map.end())
{'fraction_non_alphanumeric': 0.12359902322960366, 'fraction_numerical': 0.003882036190595454, 'mean_word_length': 3.0828220858895707, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1048949', 'n_tokens_mistral': 4849, 'n_tokens_neox': 4775, 'n_words': 1147}