<commit_before>/** \file    add_synonyms.cc
 *  \brief   Generic version for augmenting title data with synonyms found
             in the authority data
 *  \author  <NAME>
 */

/*
    Copyright (C) 2016-2018, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

/*  We offer a list of tags and subfields where the primary data resides along
    with a list of tags and subfields where the synonym data is found and
    a list of unused fields in the title data where the synonyms can be stored
*/

#include <fstream>
#include <iostream>
#include <map>
#include <vector>
#include <cstdlib>
#include "Compiler.h"
#include "MARC.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "util.h"


static unsigned modified_count(0);
static unsigned record_count(0);
const unsigned FIELD_MIN_NON_DATA_SIZE(4); // Indicator 1 + 2, unit separator and subfield code
const unsigned int NUMBER_OF_LANGUAGES(9);
const std::vector<std::string> languages_to_translate{ "en", "fr", "es", "it", "hans", "hant", "pt", "ru", "el" };
enum Languages { EN, FR, ES, IT, HANS, HANT, PT, RU, EL, LANGUAGES_END };


void Usage() {
    std::cerr << "Usage: " << ::progname << " master_marc_input norm_data_marc_input marc_output\n";
    std::exit(EXIT_FAILURE);
}


inline std::string GetTag(const std::string &tag_and_subfields_spec) {
    return tag_and_subfields_spec.substr(0, 3);
}


inline std::string GetSubfieldCodes(const std::string &tag_and_subfields_spec) {
    return tag_and_subfields_spec.substr(3);
}


bool FilterPasses(const MARC::Record &record, const std::map<std::string, std::pair<std::string, std::string>> &filter_specs, const std::string &field_spec) {
      auto filter_spec(filter_specs.find(field_spec));
      if (filter_spec == filter_specs.cend())
          return true;

      auto rule(filter_spec->second);
      // We have field_spec in key and rule to match in value
      std::string subfield_codes(GetSubfieldCodes(rule.first));
      if (subfield_codes.length() != 1)
          LOG_ERROR("in FilterPasses: Invalid subfield specification "  + subfield_codes + " for filter!");

      std::string subfield_value;
      const auto field(record.getFirstField(GetTag(rule.first)));
      if (field == record.end())
          return false;
      const std::vector<std::string> subfield_values(field->getSubfields().extractSubfields(subfield_codes[0]));
      if (subfield_values.empty())
          return false;

      return subfield_values[0] == rule.second;
}


void ExtractSynonyms(MARC::Reader * const authority_reader,
                     const std::vector<std::string> &primary_tags_and_subfield_codes,
                     const std::vector<std::string> &synonym_tags_and_subfield_codes,
                     std::vector<std::map<std::string, std::string>> * const synonym_maps,
                     const std::map<std::string, std::pair<std::string, std::string>> &filter_spec)
{
    while (const MARC::Record record = authority_reader->read()) {
        auto primary_tag_and_subfield_codes(primary_tags_and_subfield_codes.cbegin());
        auto synonym_tag_and_subfield_codes(synonym_tags_and_subfield_codes.cbegin());
        auto synonym_map(synonym_maps->begin());
        for (/*intentionally empty*/;  primary_tag_and_subfield_codes != primary_tags_and_subfield_codes.cend();
                                    ++primary_tag_and_subfield_codes, ++synonym_tag_and_subfield_codes, ++synonym_map)
        {
            // Fill maps with synonyms
            std::vector<std::string> primary_values(record.getSubfieldAndNumericSubfieldValues(GetTag(*primary_tag_and_subfield_codes), GetSubfieldCodes(*primary_tag_and_subfield_codes)));
            std::vector<std::string> synonym_values(record.getSubfieldAndNumericSubfieldValues(GetTag(*synonym_tag_and_subfield_codes), GetSubfieldCodes(*synonym_tag_and_subfield_codes)));

            if (FilterPasses(record, filter_spec, *primary_tag_and_subfield_codes) and primary_values.size() and synonym_values.size()) {
                // Append if other synonyms for the same keyword exist
                const std::string key(StringUtil::Join(primary_values, ','));
                const std::string existing_synonyms((*synonym_map)[key]);
                std::string new_synonyms(StringUtil::Join(synonym_values, ','));
                if (not existing_synonyms.empty())
                    new_synonyms += "," + existing_synonyms;
                (*synonym_map)[key] = new_synonyms;
            }
        }
    }
}


inline std::string GetMapValueOrEmptyString(const std::map<std::string, std::string> &map, const std::string &searchterm) {
    auto value(map.find(searchterm));
    return (value != map.cend()) ? value->second : "";
}


inline std::vector<std::string> GetMapValueOrEmptyString(const std::map<std::string, std::vector<std::string>> &map,
                                            const std::string &searchterm)
{
    auto value(map.find(searchterm));
    return (value != map.cend()) ? value->second : std::vector<std::string>();
}

void WriteSynonymEntry(MARC::Record * const record, std::string tag, unsigned indicator2,
                       const char subfield_code, const std::string &synonyms) {
      if (record->hasTagWithIndicators(tag, '0', indicator2 + '0'))
                    LOG_ERROR("in ProcessRecord: Could not insert field " + tag + " with indicators \'0\' and \'" +
                              std::to_string(indicator2) + "\' for PPN "  + record->getControlNumber() + '!');
      record->insertField(tag, { MARC::Subfield(subfield_code, synonyms) }, '0', indicator2 + '0');<commit_msg>Update add_synonyms.cc<commit_after>/** \file    add_synonyms.cc
 *  \brief   Generic version for augmenting title data with synonyms found
             in the authority data
 *  \author  <NAME>
 */

/*
    Copyright (C) 2016-2018, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

/*  We offer a list of tags and subfields where the primary data resides along
    with a list of tags and subfields where the synonym data is found and
    a list of unused fields in the title data where the synonyms can be stored
*/

#include <fstream>
#include <iostream>
#include <map>
#include <vector>
#include <cstdlib>
#include "Compiler.h"
#include "MARC.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "util.h"


static unsigned modified_count(0);
static unsigned record_count(0);
const unsigned FIELD_MIN_NON_DATA_SIZE(4); // Indicator 1 + 2, unit separator and subfield code
const unsigned int NUMBER_OF_LANGUAGES(9);
const std::vector<std::string> languages_to_translate{ "en", "fr", "es", "it", "hans", "hant", "pt", "ru", "el" };
enum Languages { EN, FR, ES, IT, HANS, HANT, PT, RU, EL, LANGUAGES_END };


void Usage() {
    std::cerr << "Usage: " << ::progname << " master_marc_input norm_data_marc_input marc_output\n";
    std::exit(EXIT_FAILURE);
}


inline std::string GetTag(const std::string &tag_and_subfields_spec) {
    return tag_and_subfields_spec.substr(0, 3);
}


inline std::string GetSubfieldCodes(const std::string &tag_and_subfields_spec) {
    return tag_and_subfields_spec.substr(3);
}


bool FilterPasses(const MARC::Record &record, const std::map<std::string, std::pair<std::string, std::string>> &filter_specs, const std::string &field_spec) {
      auto filter_spec(filter_specs.find(field_spec));
      if (filter_spec == filter_specs.cend())
          return true;

      auto rule(filter_spec->second);
      // We have field_spec in key and rule to match in value
      std::string subfield_codes(GetSubfieldCodes(rule.first));
      if (subfield_codes.length() != 1)
          LOG_ERROR("in FilterPasses: Invalid subfield specification "  + subfield_codes + " for filter!");

      std::string subfield_value;
      const auto field(record.getFirstField(GetTag(rule.first)));
      if (field == record.end())
          return false;
      const std::vector<std::string> subfield_values(field->getSubfields().extractSubfields(subfield_codes[0]));
      if (subfield_values.empty())
          return false;

      return subfield_values[0] == rule.second;
}


void ExtractSynonyms(MARC::Reader * const authority_reader,
                     const std::vector<std::string> &primary_tags_and_subfield_codes,
                     const std::vector<std::string> &synonym_tags_and_subfield_codes,
                     std::vector<std::map<std::string, std::string>> * const synonym_maps,
                     const std::map<std::string, std::pair<std::string, std::string>> &filter_spec)
{
    while (const MARC::Record record = authority_reader->read()) {
        auto primary_tag_and_subfield_codes(primary_tags_and_subfield_codes.cbegin());
        auto synonym_tag_and_subfield_codes(synonym_tags_and_subfield_codes.cbegin());
        auto synonym_map(synonym_maps->begin());
        for (/*intentionally empty*/;  primary_tag_and_subfield_codes != primary_tags_and_subfield_codes.cend();
                                    ++primary_tag_and_subfield_codes, ++synonym_tag_and_subfield_codes, ++synonym_map)
        {
            // Fill maps with synonyms
            std::vector<std::string> primary_values(record.getSubfieldAndNumericSubfieldValues(GetTag(*primary_tag_and_subfield_codes), GetSubfieldCodes(*primary_tag_and_subfield_codes)));
            std::vector<std::string> synonym_values(record.getSubfieldAndNumericSubfieldValues(GetTag(*synonym_tag_and_subfield_codes), GetSubfieldCodes(*synonym_tag_and_subfield_codes)));

            if (FilterPasses(record, filter_spec, *primary_tag_and_subfield_codes) and primary_values.size() and synonym_values.size()) {
                // Append if other synonyms for the same keyword exist
                const std::string key(StringUtil::Join(primary_values, ','));
                const std::string existing_synonyms((*synonym_map)[key]);
                std::string new_synonyms(StringUtil::Join(synonym_values, ','));
                if (not existing_synonyms.empty())
                    new_synonyms += "," + existing_synonyms;
                (*synonym_map)[key] = new_synonyms;
            }
        }
    }
}


inline std::string GetMapValueOrEmptyString(const std::map<std::string, std::string> &map, const std::string &searchterm) {
    auto value(map.find(searchterm));
    return (value != map.cend()) ? value->second : "";
}


inline std::vector<std::string> GetMapValueOrEmptyString(const std::map<std::string, std::vector<std::string>> &map,
                                            const std::string &searchterm)
{
    auto value(map.find(searchterm));
    return (value != map.cend()) ? value->second : std::vector<std::string>();
}

void WriteSynonymEntry(MARC::Record * const record, std::string tag, unsigned indicator2,
                       const char subfield_code, const std::string &synonyms) {
    if (record->hasTagWithIndicators(tag, '0', indicator2 + '0'))
        LOG_ERROR("in ProcessRecord: Could not insert field " + tag + " with indicators \'0\' and \'" +
                  std::to_string(indicator2) + "\' for PPN "  + record->getControlNumber() + '!');
    record->insertField(tag, { MARC::Subfield(subfield_code, synonyms) }, '0', indicator2 + '0');
{'fraction_non_alphanumeric': 0.11606358335330298, 'fraction_numerical': 0.004792715073088905, 'mean_word_length': 3.6114180478821365, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4485863', 'n_tokens_mistral': 3819, 'n_tokens_neox': 3702, 'n_words': 956}