<commit_before>/** \brief A tool to add DDC metadata to title data using various means.
 *  \author Dr. <NAME> (<EMAIL>)
 *
 *  \copyright 2015 Universitätsbiblothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <iostream>
#include <memory>
#include <set>
#include <unordered_map>
#include <cstdlib>
#include <cstring>
#include "DirectoryEntry.h"
#include "Leader.h"
#include "MarcUtil.h"
#include "RegexMatcher.h"
#include "Subfields.h"
#include "util.h"


bool IsPossibleDDC(const std::string &ddc_candidate) {
    static const RegexMatcher *matcher(RegexMatcher::RegexMatcherFactory("^\\d\\d\\d"));
    std::string err_msg;
    if (not matcher->matched(ddc_candidate, &err_msg)) {
	if (err_msg.empty())
	    return false;
	Error("unexpected regex error while trying to match \"" + ddc_candidate + "\": " + err_msg);
    }

    return true;
}


void ExtractDDCsFromField(const std::string &tag, const std::vector<DirectoryEntry> &dir_entries,
			  const std::vector<std::string> &fields, std::set<std::string> * const ddcs)
{
    const auto begin_end(DirectoryEntry::FindFields(tag, dir_entries));
    for (auto iter(begin_end.first); iter != begin_end.second; ++iter) {
	const Subfields subfields(fields[iter - dir_entries.begin()]);
	if (subfields.hasSubfield('z')) // Auxillary table number => not a regular DDC in $a!
	    continue;

	const auto subfield_a_begin_end(subfields.getIterators('a'));
	for (auto ddc(subfield_a_begin_end.first); ddc != subfield_a_begin_end.second; ++ddc) {
	    if (IsPossibleDDC(ddc->second))
		ddcs->insert(ddc->second);
	}
    }
}


void ExtractDDCsFromNormdata(const bool verbose, FILE * const norm_input,
			     std::unordered_map<std::string, std::set<std::string>> * const norm_ids_to_ddcs_map)
{
    norm_ids_to_ddcs_map->clear();
    if (verbose)
        std::cerr << "Starting loading of norm data.\n";

    unsigned count(0), ddc_record_count(0);
    while (const MarcUtil::Record record = MarcUtil::Record(norm_input)) {
        ++count;

	const std::vector<DirectoryEntry> &dir_entries(record.getDirEntries());
        const auto _001_iter(DirectoryEntry::FindField("001", dir_entries));
        if (_001_iter == dir_entries.end())
            continue;
	const std::vector<std::string> &fields(record.getFields());
        const std::string &control_number(fields[_001_iter - dir_entries.begin()]);

	std::set<std::string> ddcs;
	ExtractDDCsFromField("083", dir_entries, fields, &ddcs);
	ExtractDDCsFromField("089", dir_entries, fields, &ddcs);

	if (not ddcs.empty()) {
	    ++ddc_record_count;
	    norm_ids_to_ddcs_map->insert(std::make_pair(control_number, ddcs));
	}
    }

    if (verbose) {
        std::cerr << "Read " << count << " norm data records.\n";
        std::cerr << ddc_record_count << " records had DDC entries.\n";
    }
}


void ExtractTopicIDs(const std::string &tags, const MarcUtil::Record &record, const std::set<std::string> &existing_ddcs,
		     std::set<std::string> * const topic_ids)
{
    topic_ids->clear();

    std::vector<std::string> individual_tags;
    StringUtil::Split(tags, ':', &individual_tags);

    const std::vector<DirectoryEntry> &dir_entries(record.getDirEntries());
    const std::vector<std::string> &fields(record.getFields());
    for (const auto &tag : individual_tags) {
        const ssize_t first_index(record.getFieldIndex(tag));
        if (first_index == -1)
            continue;

        for (size_t index(first_index); index < dir_entries.size() and dir_entries[index].getTag() == tag; ++index) {
            const Subfields subfields(fields[index]);
            const auto begin_end(subfields.getIterators('0'));
            for (auto subfield0(begin_end.first); subfield0 != begin_end.second; ++subfield0) {
                if (not StringUtil::StartsWith(subfield0->second, "(DE-576)"))
                    continue;

                const std::string topic_id(subfield0->second.substr(8));
		if (existing_ddcs.find(topic_id) == existing_ddcs.end()) // This one is new!
		    topic_ids->insert(topic_id);
            }
        }
    }
}


void AugmentRecordsWithDDCs(const bool verbose, FILE * const title_input, FILE * const title_output,
			    const std::unordered_map<std::string, std::set<std::string>> &norm_ids_to_ddcs_map)
{
    if (verbose)
        std::cerr << "Starting augmenting of data.\n";

    unsigned count(0), augmented_count(0), already_had_ddcs(0), never_had_ddcs_and_now_have_ddcs(0);
    while (MarcUtil::Record record = MarcUtil::Record(title_input)) {
        ++count;

	// Extract already existing DDCs:
	std::set<std::string> existing_ddcs;
	const std::vector<DirectoryEntry> &dir_entries(record.getDirEntries());
	const std::vector<std::string> &fields(record.getFields());
	ExtractDDCsFromField("082", dir_entries, fields, &existing_ddcs);
	ExtractDDCsFromField("083", dir_entries, fields, &existing_ddcs);
	if (not existing_ddcs.empty())
	    ++already_had_ddcs;
	
	std::set<std::string> topic_ids; // = the IDs of the corresponding norm data records
	ExtractTopicIDs("fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b", record, existing_ddcs, &topic_ids);
	if (topic_ids.empty()) {
	    record.write(title_output);
	    continue;
	}

	std::set<std::string> new_ddcs;
	for (const auto &topic_id : topic_ids) {
	    const auto iter(norm_ids_to_ddcs_map.find(topic_id));
	    if (iter != norm_ids_to_ddcs_map.end())
		new_ddcs.insert(iter->second.begin(), iter->second.end());
	}

	if (not new_ddcs.empty()) {
	    ++augmented_count;
	    if (existing_ddcs.empty())
		++never_had_ddcs_and_now_have_ddcs;
	    for (const auto &new_ddc : new_ddcs) {
		const std::string new_field("0 ""\x1F""a" + new_ddc);
		record.insertField("082", new_field);
	    }
	}

	record.write(title_output);
    }<commit_msg>Add a tag in 082 for the newly added DDC's.<commit_after>/** \brief A tool to add DDC metadata to title data using various means.
 *  \author Dr. <NAME> (<EMAIL>)
 *
 *  \copyright 2015 Universitätsbiblothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <iostream>
#include <memory>
#include <set>
#include <unordered_map>
#include <cstdlib>
#include <cstring>
#include "DirectoryEntry.h"
#include "Leader.h"
#include "MarcUtil.h"
#include "RegexMatcher.h"
#include "Subfields.h"
#include "util.h"


bool IsPossibleDDC(const std::string &ddc_candidate) {
    static const RegexMatcher *matcher(RegexMatcher::RegexMatcherFactory("^\\d\\d\\d"));
    std::string err_msg;
    if (not matcher->matched(ddc_candidate, &err_msg)) {
	if (err_msg.empty())
	    return false;
	Error("unexpected regex error while trying to match \"" + ddc_candidate + "\": " + err_msg);
    }

    return true;
}


void ExtractDDCsFromField(const std::string &tag, const std::vector<DirectoryEntry> &dir_entries,
			  const std::vector<std::string> &fields, std::set<std::string> * const ddcs)
{
    const auto begin_end(DirectoryEntry::FindFields(tag, dir_entries));
    for (auto iter(begin_end.first); iter != begin_end.second; ++iter) {
	const Subfields subfields(fields[iter - dir_entries.begin()]);
	if (subfields.hasSubfield('z')) // Auxillary table number => not a regular DDC in $a!
	    continue;

	const auto subfield_a_begin_end(subfields.getIterators('a'));
	for (auto ddc(subfield_a_begin_end.first); ddc != subfield_a_begin_end.second; ++ddc) {
	    if (IsPossibleDDC(ddc->second))
		ddcs->insert(ddc->second);
	}
    }
}


void ExtractDDCsFromNormdata(const bool verbose, FILE * const norm_input,
			     std::unordered_map<std::string, std::set<std::string>> * const norm_ids_to_ddcs_map)
{
    norm_ids_to_ddcs_map->clear();
    if (verbose)
        std::cerr << "Starting loading of norm data.\n";

    unsigned count(0), ddc_record_count(0);
    while (const MarcUtil::Record record = MarcUtil::Record(norm_input)) {
        ++count;

	const std::vector<DirectoryEntry> &dir_entries(record.getDirEntries());
        const auto _001_iter(DirectoryEntry::FindField("001", dir_entries));
        if (_001_iter == dir_entries.end())
            continue;
	const std::vector<std::string> &fields(record.getFields());
        const std::string &control_number(fields[_001_iter - dir_entries.begin()]);

	std::set<std::string> ddcs;
	ExtractDDCsFromField("083", dir_entries, fields, &ddcs);
	ExtractDDCsFromField("089", dir_entries, fields, &ddcs);

	if (not ddcs.empty()) {
	    ++ddc_record_count;
	    norm_ids_to_ddcs_map->insert(std::make_pair(control_number, ddcs));
	}
    }

    if (verbose) {
        std::cerr << "Read " << count << " norm data records.\n";
        std::cerr << ddc_record_count << " records had DDC entries.\n";
    }
}


void ExtractTopicIDs(const std::string &tags, const MarcUtil::Record &record, const std::set<std::string> &existing_ddcs,
		     std::set<std::string> * const topic_ids)
{
    topic_ids->clear();

    std::vector<std::string> individual_tags;
    StringUtil::Split(tags, ':', &individual_tags);

    const std::vector<DirectoryEntry> &dir_entries(record.getDirEntries());
    const std::vector<std::string> &fields(record.getFields());
    for (const auto &tag : individual_tags) {
        const ssize_t first_index(record.getFieldIndex(tag));
        if (first_index == -1)
            continue;

        for (size_t index(first_index); index < dir_entries.size() and dir_entries[index].getTag() == tag; ++index) {
            const Subfields subfields(fields[index]);
            const auto begin_end(subfields.getIterators('0'));
            for (auto subfield0(begin_end.first); subfield0 != begin_end.second; ++subfield0) {
                if (not StringUtil::StartsWith(subfield0->second, "(DE-576)"))
                    continue;

                const std::string topic_id(subfield0->second.substr(8));
		if (existing_ddcs.find(topic_id) == existing_ddcs.end()) // This one is new!
		    topic_ids->insert(topic_id);
            }
        }
    }
}


void AugmentRecordsWithDDCs(const bool verbose, FILE * const title_input, FILE * const title_output,
			    const std::unordered_map<std::string, std::set<std::string>> &norm_ids_to_ddcs_map)
{
    if (verbose)
        std::cerr << "Starting augmenting of data.\n";

    unsigned count(0), augmented_count(0), already_had_ddcs(0), never_had_ddcs_and_now_have_ddcs(0);
    while (MarcUtil::Record record = MarcUtil::Record(title_input)) {
        ++count;

	// Extract already existing DDCs:
	std::set<std::string> existing_ddcs;
	const std::vector<DirectoryEntry> &dir_entries(record.getDirEntries());
	const std::vector<std::string> &fields(record.getFields());
	ExtractDDCsFromField("082", dir_entries, fields, &existing_ddcs);
	ExtractDDCsFromField("083", dir_entries, fields, &existing_ddcs);
	if (not existing_ddcs.empty())
	    ++already_had_ddcs;
	
	std::set<std::string> topic_ids; // = the IDs of the corresponding norm data records
	ExtractTopicIDs("fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b", record, existing_ddcs, &topic_ids);
	if (topic_ids.empty()) {
	    record.write(title_output);
	    continue;
	}

	std::set<std::string> new_ddcs;
	for (const auto &topic_id : topic_ids) {
	    const auto iter(norm_ids_to_ddcs_map.find(topic_id));
	    if (iter != norm_ids_to_ddcs_map.end())
		new_ddcs.insert(iter->second.begin(), iter->second.end());
	}

	if (not new_ddcs.empty()) {
	    ++augmented_count;
	    if (existing_ddcs.empty())
		++never_had_ddcs_and_now_have_ddcs;
	    for (const auto &new_ddc : new_ddcs) {
		const std::string new_field("0 ""\x1F""a" + new_ddc + "\x1F""cfrom_topic_norm_data");
		record.insertField("082", new_field);
	    }
	}

	record.write(title_output);
    }
{'fraction_non_alphanumeric': 0.13791498603785293, 'fraction_numerical': 0.011169717654359293, 'mean_word_length': 4.1987903225806456, 'pattern_counts': {'":': 2, '<': 91, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3578960', 'n_tokens_mistral': 4474, 'n_tokens_neox': 4229, 'n_words': 988}