<commit_before>/** \file    augment_authority_data_with_translations.cc
 *  \brief   Extract translations from database amd augment the normdata file
 *  \author  <NAME>
 */

/*
    Copyright (C) 2016, 2017 Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <string>
#include <vector>
#include <tuple>
#include <unordered_set>
#include <utility>
#include <cstdlib>
#include "Compiler.h"
#include "DbConnection.h"
#include "DbResultSet.h"
#include "DbRow.h"
#include "IniFile.h"
#include "MarcReader.h"
#include "MarcRecord.h"
#include "MarcWriter.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "util.h"


namespace {


// Save language code, translation, origin, status
typedef std::tuple<std::string, std::string, std::string, std::string> OneTranslation;


static unsigned record_count(0);
static unsigned modified_count(0);


void Usage() {
    std::cerr << "Usage: " << ::progname << " authority_data_input authority_data_output\n";
    std::exit(EXIT_FAILURE);
}


inline bool IsReliableSynonym(const std::string &status) {
    return status == "replaced_synonym" or status == "new_synonym" or status == "derived_synonym";
}


std::string ReplaceAngleBracketsWithParentheses(const std::string &value) {
    return StringUtil::Map(value, "<>", "()");
}


void ExtractTranslations(DbConnection * const db_connection, std::map<std::string,
                         std::vector<OneTranslation> > * const all_translations)
{
    db_connection->queryOrDie("SELECT DISTINCT ppn FROM keyword_translations");
    DbResultSet ppn_result_set(db_connection->getLastResultSet());
    while (const DbRow ppn_row = ppn_result_set.getNextRow()) {
        std::string ppn = ppn_row["ppn"];
        db_connection->queryOrDie("SELECT ppn, language_code, translation, origin, status FROM keyword_translations "
                                  "WHERE ppn='" + ppn + "'");
        DbResultSet result_set(db_connection->getLastResultSet());
        std::vector<OneTranslation> translations;
        while (const DbRow row = result_set.getNextRow()) {
            // We are not interested in synonym fields as we will directly derive synonyms from the translation field
            // Furthermore we insert keywords where the german translation is the reference and needs no further
            // inserting
            if (not IsReliableSynonym(row["status"]) and row["language_code"] != "ger") {
                std::string translation(row["translation"]);
                // Handle '#'-separated synonyms appropriately
                if (translation.find("#") == std::string::npos and not translation.empty())
                    translations.emplace_back(ReplaceAngleBracketsWithParentheses(translation),
                                              row["language_code"], row["origin"], row["status"]);
                else {
                    std::vector<std::string> primary_and_synonyms;
                    StringUtil::SplitThenTrim(translation, "#", " \t\n", &primary_and_synonyms);
		    // Use the first translation as non-synonmym
                    if (primary_and_synonyms.size() > 0) {
                        translations.emplace_back(primary_and_synonyms[0], row["language_code"], row["origin"],
                                              row["status"]);
                        // Add further synonyms as derived synonyms
                        for (auto synonyms(std::next(primary_and_synonyms.cbegin()));
                            synonyms != primary_and_synonyms.cend(); ++synonyms)
                            translations.emplace_back(ReplaceAngleBracketsWithParentheses(*synonyms),
                                                  row["language_code"], row["origin"], "derived_synonym");
                    }
                }
            }
        }
        all_translations->insert(std::make_pair(ppn, translations));
    }
}


void InsertTranslation(MarcRecord * const record, const char indicator1, const char indicator2,
                       const std::string &term, const std::string &language_code, const std::string &status)
{
    Subfields subfields(indicator1, indicator2);
    subfields.addSubfield('a', term);
    subfields.addSubfield('9', "L:" + language_code);
    subfields.addSubfield('9', "Z:" + std::string(IsReliableSynonym(status) ? "VW" : "AF"));
    subfields.addSubfield('2', "IxTheo");
    record->insertField("750", subfields);
}


size_t GetFieldIndexForExistingTranslation(const MarcRecord *record, const std::vector<size_t> &field_indices,
                                           const std::string &language_code, const std::string &status) {
    // We can have several either previously existing or already inserted synonyms, so don't replace synonyms
    if (IsReliableSynonym(status))
        return MarcRecord::FIELD_NOT_FOUND;

    for (auto field_index : field_indices) {
        const Subfields subfields(record->getSubfields(field_index));
        if (subfields.hasSubfieldWithValue('2', "IxTheo") and
            subfields.hasSubfieldWithValue('9', "L:" + language_code) and
            subfields.hasSubfieldWithValue('9', "Z:AF"))
                return field_index;
    }
    return MarcRecord::FIELD_NOT_FOUND;<commit_msg>Resolving conflict<commit_after>/** \file    augment_authority_data_with_translations.cc
 *  \brief   Extract translations from database amd augment the normdata file
 *  \author  <NAME>
 */

/*
    Copyright (C) 2016, 2017 Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <string>
#include <vector>
#include <tuple>
#include <unordered_set>
#include <utility>
#include <cstdlib>
#include "Compiler.h"
#include "DbConnection.h"
#include "DbResultSet.h"
#include "DbRow.h"
#include "IniFile.h"
#include "MarcReader.h"
#include "MarcRecord.h"
#include "MarcWriter.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "util.h"


namespace {


// Save language code, translation, origin, status
typedef std::tuple<std::string, std::string, std::string, std::string> OneTranslation;


static unsigned record_count(0);
static unsigned modified_count(0);


void Usage() {
    std::cerr << "Usage: " << ::progname << " authority_data_input authority_data_output\n";
    std::exit(EXIT_FAILURE);
}


inline bool IsReliableSynonym(const std::string &status) {
    return status == "replaced_synonym" or status == "new_synonym" or status == "derived_synonym";
}


std::string ReplaceAngleBracketsWithParentheses(const std::string &value) {
    return StringUtil::Map(value, "<>", "()");
}


void ExtractTranslations(DbConnection * const db_connection, std::map<std::string,
                         std::vector<OneTranslation> > * const all_translations)
{
    db_connection->queryOrDie("SELECT DISTINCT ppn FROM keyword_translations");
    DbResultSet ppn_result_set(db_connection->getLastResultSet());
    while (const DbRow ppn_row = ppn_result_set.getNextRow()) {
        std::string ppn = ppn_row["ppn"];
        db_connection->queryOrDie("SELECT ppn, language_code, translation, origin, status FROM keyword_translations "
                                  "WHERE ppn='" + ppn + "'");
        DbResultSet result_set(db_connection->getLastResultSet());
        std::vector<OneTranslation> translations;
        while (const DbRow row = result_set.getNextRow()) {
            // We are not interested in synonym fields as we will directly derive synonyms from the translation field
            // Furthermore we insert keywords where the german translation is the reference and needs no further
            // inserting
            if (not IsReliableSynonym(row["status"]) and row["language_code"] != "ger") {
                std::string translation(row["translation"]);
                // Handle '#'-separated synonyms appropriately
                if (translation.find("#") == std::string::npos and not translation.empty())
                    translations.emplace_back(ReplaceAngleBracketsWithParentheses(translation),
                                              row["language_code"], row["origin"], row["status"]);
                else {
                    std::vector<std::string> primary_and_synonyms;
                    StringUtil::SplitThenTrim(translation, "#", " \t\n", &primary_and_synonyms);
		    // Use the first translation as non-synonmym
                    if (primary_and_synonyms.size() > 0) {
                        translations.emplace_back(primary_and_synonyms[0], row["language_code"], row["origin"],
                                              row["status"]);
                        // Add further synonyms as derived synonyms
                        for (auto synonyms(std::next(primary_and_synonyms.cbegin()));
                            synonyms != primary_and_synonyms.cend(); ++synonyms)
                            translations.emplace_back(ReplaceAngleBracketsWithParentheses(*synonyms),
                                                  row["language_code"], row["origin"], "derived_synonym");
                    }
                }
            }
        }
        all_translations->insert(std::make_pair(ppn, translations));
    }
}


void InsertTranslation(MarcRecord * const record, const char indicator1, const char indicator2,
                       const std::string &term, const std::string &language_code, const std::string &status)
{
    Subfields subfields(indicator1, indicator2);
    subfields.addSubfield('a', term);
    subfields.addSubfield('9', "L:" + language_code);
    subfields.addSubfield('9', "Z:" + std::string(IsReliableSynonym(status) ? "VW" : "AF"));
    subfields.addSubfield('2', "IxTheo");
    record->insertField("750", subfields);
}


size_t GetFieldIndexForExistingTranslation(const MarcRecord *record, const std::vector<size_t> &field_indices,
                                           const std::string &language_code, const std::string &status) {
    // We can have several either previously existing or already inserted synonyms, so don't replace synonyms
    if (IsReliableSynonym(status))
        return MarcRecord::FIELD_NOT_FOUND;
    for (const auto field_index : field_indices) {
        Subfields subfields_present(record->getSubfields(field_index));
        if (subfields_present.hasSubfieldWithValue('2', "IxTheo") and
            subfields_present.hasSubfieldWithValue('9', "L:" + language_code) and
            subfields_present.hasSubfieldWithValue('9', "Z:AF"))
                return field_index;
    }
    return MarcRecord::FIELD_NOT_FOUND;
{'fraction_non_alphanumeric': 0.10791610284167795, 'fraction_numerical': 0.004397834912043302, 'mean_word_length': 2.948247078464107, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1880049', 'n_tokens_mistral': 3332, 'n_tokens_neox': 3171, 'n_words': 868}