<commit_before>/** \file    augment_time_aspects.cc
 *  \brief   A tool for adding normalised time references to MARC-21 datasets.
 *  \author  Dr. <NAME>
 */

/*
    Copyright (C) 2019-2021, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <unordered_map>
#include <cstdlib>
#include "Compiler.h"
#include "MARC.h"
#include "RangeUtil.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "util.h"


namespace {

static const std::vector<std::string> TIME_ASPECT_GND_LINKING_TAGS{ "689" };
static const std::vector<std::string> KEYWORD_PREFIXES{ "Geschichte ", "Geistesgeschichte ", "Ideengeschichte ", "Kirchengeschichte ",
                                                         "Sozialgeschichte ", "Vor- und Frühgeschichte ", "Weltgeschichte ", "Prognose " };


inline std::vector<std::string>::const_iterator FindFirstPrefixMatch(const std::string &s, const std::vector<std::string> &prefixes) {
    for (auto prefix(prefixes.cbegin()); prefix != prefixes.cend(); ++prefix) {
        if (StringUtil::StartsWith(s, *prefix))
            return prefix;
    }

    return prefixes.cend();
}


void LoadAuthorityData(MARC::Reader * const reader,
                       std::unordered_map<std::string, std::string> * const authority_ppns_to_time_codes_map,
                       std::unordered_map<std::string, std::string> * const authority_ppns_to_time_categories_map)
{
    unsigned total_count(0);
    while (auto record = reader->read()) {
        ++total_count;

        bool found(false);
        const auto _548_field(record.findTag("548"));
        if (_548_field != record.end() and _548_field->hasSubfieldWithValue('i', "Zeitraum")) {
            const std::string free_form_range_candidate(_548_field->getFirstSubfieldWithCode('a'));
            std::string range;
            if (RangeUtil::ConvertTextToTimeRange(free_form_range_candidate, &range, /* special_case_centuries = */true)) {
                (*authority_ppns_to_time_codes_map)[record.getControlNumber()] = range;
                found = true;
                const auto _150_field(record.findTag("150"));
                if (_150_field != record.end() and _150_field->hasSubfield('a')) {
                    std::string _150a_subfield = _150_field->getFirstSubfieldWithCode('a');
                    if (StringUtil::Contains(_150a_subfield, free_form_range_candidate) == false) 
                        _150a_subfield = _150a_subfield + " " + free_form_range_candidate;
                    (*authority_ppns_to_time_categories_map)[record.getControlNumber()] = _150a_subfield;
                }
            }
            else
                LOG_WARNING("can't convert \"" + free_form_range_candidate + "\" to a time range!");
        } 
        if (found == false) {
            const auto _450_field(record.findTag("450"));
            if (_450_field != record.end() and _450_field->hasSubfield('a')) {
                std::string _450a_subfield = _450_field->getFirstSubfieldWithCode('a');
                const auto matched_prefix(FindFirstPrefixMatch(_450a_subfield, KEYWORD_PREFIXES));
                if (matched_prefix != KEYWORD_PREFIXES.cend()) {
                    std::string range;
                    if (RangeUtil::ConvertTextToTimeRange(_450a_subfield.substr(matched_prefix->length()), &range)) {
                        (*authority_ppns_to_time_codes_map)[record.getControlNumber()] = range;
                        (*authority_ppns_to_time_categories_map)[record.getControlNumber()] = _450a_subfield;
                    }
                }
            }
        }
    }

    LOG_INFO("found " + std::to_string(authority_ppns_to_time_codes_map->size()) + " time aspect records among "
             + std::to_string(total_count) + " authority records.");<commit_msg>build loops around fields 548 and 450<commit_after>/** \file    augment_time_aspects.cc
 *  \brief   A tool for adding normalised time references to MARC-21 datasets.
 *  \author  Dr. <NAME>
 */

/*
    Copyright (C) 2019-2021, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <unordered_map>
#include <cstdlib>
#include "Compiler.h"
#include "MARC.h"
#include "RangeUtil.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "util.h"


namespace {

static const std::vector<std::string> TIME_ASPECT_GND_LINKING_TAGS{ "689" };
static const std::vector<std::string> KEYWORD_PREFIXES{ "Geschichte ", "Geistesgeschichte ", "Ideengeschichte ", "Kirchengeschichte ",
                                                         "Sozialgeschichte ", "Vor- und Frühgeschichte ", "Weltgeschichte ", "Prognose " };


inline std::vector<std::string>::const_iterator FindFirstPrefixMatch(const std::string &s, const std::vector<std::string> &prefixes) {
    for (auto prefix(prefixes.cbegin()); prefix != prefixes.cend(); ++prefix) {
        if (StringUtil::StartsWith(s, *prefix))
            return prefix;
    }

    return prefixes.cend();
}


void LoadAuthorityData(MARC::Reader * const reader,
                       std::unordered_map<std::string, std::string> * const authority_ppns_to_time_codes_map,
                       std::unordered_map<std::string, std::string> * const authority_ppns_to_time_categories_map)
{
    unsigned total_count(0);
    while (auto record = reader->read()) {
        ++total_count;

        bool found_548(false);

        auto _548_field(record.findTag("548"));
        while (_548_field != record.end()) {
            if (_548_field->hasSubfieldWithValue('i', "Zeitraum")) {
                const std::string free_form_range_candidate(_548_field->getFirstSubfieldWithCode('a'));
                std::string range;
                if (RangeUtil::ConvertTextToTimeRange(free_form_range_candidate, &range, /* special_case_centuries = */ true)) {
                    (*authority_ppns_to_time_codes_map)[record.getControlNumber()] = range;
                    found_548 = true;
                    const auto _150_field(record.findTag("150"));
                    if (_150_field != record.end() and _150_field->hasSubfield('a')) {
                        std::string _150a_subfield = _150_field->getFirstSubfieldWithCode('a');
                        _150a_subfield = _150a_subfield + " " + free_form_range_candidate;
                        (*authority_ppns_to_time_categories_map)[record.getControlNumber()] = _150a_subfield;
                    }
                    break;
                } else
                    LOG_WARNING("can't convert \"" + free_form_range_candidate + "\" to a time range!");
            }
            _548_field++;
        }
        if (found_548 == false) {
            auto _450_field(record.findTag("450"));
            while (_450_field != record.end()) {
                if (_450_field->hasSubfield('a')) {
                    std::string _450a_subfield = _450_field->getFirstSubfieldWithCode('a');
                    const auto matched_prefix(FindFirstPrefixMatch(_450a_subfield, KEYWORD_PREFIXES));
                    if (matched_prefix != KEYWORD_PREFIXES.cend()) {
                        std::string range;
                        if (RangeUtil::ConvertTextToTimeRange(_450a_subfield.substr(matched_prefix->length()), &range)) {
                            (*authority_ppns_to_time_codes_map)[record.getControlNumber()] = range;
                            (*authority_ppns_to_time_categories_map)[record.getControlNumber()] = _450a_subfield;
                            break;
                        }
                    }
                }
                _450_field++;
            }
        }
    }

    LOG_INFO("found " + std::to_string(authority_ppns_to_time_codes_map->size()) + " time aspect records among "
             + std::to_string(total_count) + " authority records.");
{'fraction_non_alphanumeric': 0.10801585553842766, 'fraction_numerical': 0.021140717903545474, 'mean_word_length': 2.584451460142068, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4212454', 'n_tokens_mistral': 2741, 'n_tokens_neox': 2546, 'n_words': 595}