<commit_before>/** \brief A MARC-21 filter utility that can remove characters from a set of subfields.
 *  \author Dr. <NAME> (<EMAIL>)
 *
 *  \copyright 2016 Universitätsbiblothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <iostream>
#include <memory>
#include <unordered_set>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include "DirectoryEntry.h"
#include "FileUtil.h"
#include "Leader.h"
#include "MarcUtil.h"
#include "MarcXmlWriter.h"
#include "MediaTypeUtil.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "util.h"


void Usage() {
    std::cerr << "usage: " << progname << " marc_input marc_output subfield_spec1:subfield_spec2:...:subfield_specN "
              << " characters_to_delete\n"
              << "       where \"subfieldspec\" must be a MARC tag followed by a single-character\n"
              << "       subfield code and \"characters_to_delete\" is list of characters that will be removed\n"
              << "       from the contents of the specified subfields.\n\n";

    std::exit(EXIT_FAILURE);
}


std::string GetSubfieldCodes(const std::string &tag, const std::vector<std::string> &subfield_specs) {
    std::string subfield_codes;
    
    for (const auto &subfield_spec : subfield_specs) {
        if (subfield_spec.substr(0, DirectoryEntry::TAG_LENGTH) == tag)
            subfield_codes += subfield_spec[DirectoryEntry::TAG_LENGTH];
    }

    return subfield_codes;
}


void Filter(File * const input, File * const output, const std::vector<std::string> &subfield_specs,
            const std::string &filter_chars)
{
    MarcXmlWriter xml_writer(output);

    unsigned total_count(0), modified_record_count(0), modified_field_count(0);
    while (MarcUtil::Record record = MarcUtil::Record::XmlFactory(input)) {
        ++total_count;
        record.setRecordWillBeWrittenAsXml(true);

        const std::vector<DirectoryEntry> &dir_entries(record.getDirEntries());
        const std::vector<std::string> &fields(record.getFields());

        for (std::vector<DirectoryEntry>::const_iterator dir_entry(dir_entries.cbegin());
             dir_entry != dir_entries.cend(); ++dir_entry)
        {
            const std::string subfield_codes(GetSubfieldCodes(dir_entry->getTag(), subfield_specs));
            if (subfield_codes.empty()) {
                record.write(&xml_writer);
                continue;
            }

            bool modified_at_least_one(false);
            const auto field_index(dir_entry - dir_entries.cbegin());
            Subfields subfields(fields[field_index]);
            for (const auto subfield_code : subfield_codes) {
                const auto begin_end(subfields.getIterators(subfield_code));
                for (auto subfield(begin_end.first); subfield != begin_end.second; ++subfield) {
                    const auto old_length(subfield->second.length());
                    StringUtil::RemoveChars(filter_chars, &(subfield->second));
                    if (subfield->second.length() != old_length) {
                        ++modified_field_count;
                        modified_at_least_one = true;
                    }
                }
            }

            if (modified_at_least_one) {
                record.replaceField(field_index, subfields.toString());
                ++modified_record_count;
            }

            record.write(&xml_writer);
        }
    }
    
    std::cerr << "Modified " << modified_record_count << " (" << modified_field_count << " fields) of "
              << total_count << " record(s).\n";
}


// Sanity check.
bool ArePlausibleSubfieldSpecs(const std::vector<std::string> &subfield_specs) {
    if (subfield_specs.empty())
        return false;<commit_msg>Fixed some fatally flawed logic which led to a single record being written out multiple times.<commit_after>/** \brief A MARC-21 filter utility that can remove characters from a set of subfields.
 *  \author Dr. <NAME> (<EMAIL>)
 *
 *  \copyright 2016 Universitätsbiblothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <iostream>
#include <memory>
#include <unordered_set>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include "DirectoryEntry.h"
#include "FileUtil.h"
#include "Leader.h"
#include "MarcUtil.h"
#include "MarcXmlWriter.h"
#include "MediaTypeUtil.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "util.h"


void Usage() {
    std::cerr << "usage: " << progname << " marc_input marc_output subfield_spec1:subfield_spec2:...:subfield_specN "
              << " characters_to_delete\n"
              << "       where \"subfieldspec\" must be a MARC tag followed by a single-character\n"
              << "       subfield code and \"characters_to_delete\" is list of characters that will be removed\n"
              << "       from the contents of the specified subfields.\n\n";

    std::exit(EXIT_FAILURE);
}


std::string GetSubfieldCodes(const std::string &tag, const std::vector<std::string> &subfield_specs) {
    std::string subfield_codes;
    
    for (const auto &subfield_spec : subfield_specs) {
        if (subfield_spec.substr(0, DirectoryEntry::TAG_LENGTH) == tag)
            subfield_codes += subfield_spec[DirectoryEntry::TAG_LENGTH];
    }

    return subfield_codes;
}


void Filter(File * const input, File * const output, const std::vector<std::string> &subfield_specs,
            const std::string &filter_chars)
{
    MarcXmlWriter xml_writer(output);

    unsigned total_count(0), modified_record_count(0), modified_field_count(0);
    while (MarcUtil::Record record = MarcUtil::Record::XmlFactory(input)) {
        ++total_count;
        record.setRecordWillBeWrittenAsXml(true);

        const std::vector<DirectoryEntry> &dir_entries(record.getDirEntries());
        const std::vector<std::string> &fields(record.getFields());

        bool modified_at_least_one_field(false);
        for (std::vector<DirectoryEntry>::const_iterator dir_entry(dir_entries.cbegin());
             dir_entry != dir_entries.cend(); ++dir_entry)
        {
            const std::string subfield_codes(GetSubfieldCodes(dir_entry->getTag(), subfield_specs));
            if (subfield_codes.empty())
                continue;

            bool modified_at_least_one_subfield(false);
            const auto field_index(dir_entry - dir_entries.cbegin());
            Subfields subfields(fields[field_index]);
            for (const auto subfield_code : subfield_codes) {
                const auto begin_end(subfields.getIterators(subfield_code));
                for (auto subfield(begin_end.first); subfield != begin_end.second; ++subfield) {
                    const auto old_length(subfield->second.length());
                    StringUtil::RemoveChars(filter_chars, &(subfield->second));
                    if (subfield->second.length() != old_length) {
                        ++modified_field_count;
                        modified_at_least_one_subfield = true;
                    }
                }
            }

            if (modified_at_least_one_subfield) {
                modified_at_least_one_field = true;
                record.replaceField(field_index, subfields.toString());
            }
        }

        if (modified_at_least_one_field)
            ++modified_record_count;
        record.write(&xml_writer);
    }
    
    std::cerr << "Modified " << modified_record_count << " (" << modified_field_count << " fields) of "
              << total_count << " record(s).\n";
}


// Sanity check.
bool ArePlausibleSubfieldSpecs(const std::vector<std::string> &subfield_specs) {
    if (subfield_specs.empty())
        return false;
{'fraction_non_alphanumeric': 0.10921388180507817, 'fraction_numerical': 0.0028828029715046014, 'mean_word_length': 2.88458225667528, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1841701', 'n_tokens_mistral': 2662, 'n_tokens_neox': 2569, 'n_words': 667}