<commit_before>/** \brief Generates a list of values from LOK $a where $0=689.
 *  \author Dr. <NAME> (<EMAIL>)
 *
 *  \copyright 2017 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <algorithm>
#include <iostream>
#include <unordered_set>
#include <vector>
#include <cstdlib>
#include "MarcRecord.h"
#include "MarcReader.h"
#include "Subfields.h"
#include "util.h"


void Usage() {
    std::cerr << "usage: " << ::progname << " marc_input\n";
    std::exit(EXIT_FAILURE);
}


void ExtractLocalKeywords(MarcReader * const marc_reader, std::unordered_set<std::string> * const local_keywords) {
    unsigned total_count(0), matched_count(0), records_with_local_data_count(0);
    while (const MarcRecord record = marc_reader->read()) {
        ++total_count;

        std::vector<size_t> field_indices;
        if (not record.getFieldIndices("LOK", &field_indices))
            continue;
        ++records_with_local_data_count;

        bool matched(false);
        for (size_t field_index : field_indices) {
            const Subfields subfields(record.getFieldData(field_index));
            const std::string subfield0(subfields.getFirstSubfieldValue('0'));
            if (subfield0.empty() or subfield0 != "689  ")
                continue;
            std::vector<std::string> subfield_a_values;
            subfields.extractSubfields('a', &subfield_a_values);
            for (const auto &subfield_a_value : subfield_a_values) {
                local_keywords->insert(subfield_a_value);
                matched = true;
            }
        }
        if (matched)
            ++matched_count;
    }

    std::cerr << "Processed a total of " << total_count << " record(s) of which " << records_with_local_data_count
              << " had local data.\n";
    std::cerr << "Found " << matched_count << " record(s) w/ local keywords.\n";
}


void DisplayKeywords(const std::unordered_set<std::string> &local_keywords) {
    std::vector<std::string> sorted_keywords;
    sorted_keywords.reserve(local_keywords.size());

    std::copy(local_keywords.cbegin(), local_keywords.cend(), std::back_inserter(sorted_keywords));
    std::sort(sorted_keywords.begin(), sorted_keywords.end());

    for (const auto &keyword : sorted_keywords)
        std::cout << keyword << '\n';
}


int main(int argc, char *argv[]) {
    ::progname = argv[0];
    if (argc != 2)
        Usage();

    std::unique_ptr<MarcReader> marc_reader(MarcReader::Factory(argv[1]));

    try {
        std::unordered_set<std::string> local_keywords;
        ExtractLocalKeywords(marc_reader.get(), &local_keywords);
        DisplayKeywords(local_keywords);
    } catch (const std::exception &x) {
        logger->error("caught exception: " + std::string(x.what()));
    }
}<commit_msg>Converted to the new MARC API.<commit_after>/** \brief Generates a list of values from LOK $a where $0=689.
 *  \author <NAME> (<EMAIL>)
 *
 *  \copyright 2017,2018 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <algorithm>
#include <iostream>
#include <unordered_set>
#include <vector>
#include <cstdlib>
#include "MARC.h"
#include "util.h"


namespace {


[[noreturn]] void Usage() {
    std::cerr << "usage: " << ::progname << " marc_input\n";
    std::exit(EXIT_FAILURE);
}


void ExtractLocalKeywords(MARC::Reader * const marc_reader, std::unordered_set<std::string> * const local_keywords) {
    unsigned total_count(0), matched_count(0), records_with_local_data_count(0);
    while (const MARC::Record record = marc_reader->read()) {
        ++total_count;

        bool matched(false);
        for (const auto &local_field : record.getTagRange("LOK")) {
            if (local_field.getLocalTag() != "689" or local_field.getLocalIndicator1() != ' ' or local_field.getLocalIndicator2() != ' ')
                continue;

            const MARC::Subfields subfields(local_field.getSubfields());
            for (const auto &subfield : subfields) {
                if (subfield.code_ == 'a') {
                    local_keywords->insert(subfield.value_);
                    matched = true;
                }
            }
        }
        if (matched)
            ++matched_count;
    }

    LOG_INFO("Processed a total of " + std::to_string(total_count) + " record(s) of which " + std::to_string(records_with_local_data_count)
             + " had local data.");
    LOG_INFO("Found " + std::to_string(matched_count) + " record(s) w/ local keywords.");
}


void DisplayKeywords(const std::unordered_set<std::string> &local_keywords) {
    std::vector<std::string> sorted_keywords;
    sorted_keywords.reserve(local_keywords.size());

    std::copy(local_keywords.cbegin(), local_keywords.cend(), std::back_inserter(sorted_keywords));
    std::sort(sorted_keywords.begin(), sorted_keywords.end());

    for (const auto &keyword : sorted_keywords)
        std::cout << keyword << '\n';
}


} // unnamed namespace


int Main(int argc, char *argv[]) {
    if (argc != 2)
        Usage();

    auto marc_reader(MARC::Reader::Factory(argv[1]));

    std::unordered_set<std::string> local_keywords;
    ExtractLocalKeywords(marc_reader.get(), &local_keywords);
    DisplayKeywords(local_keywords);

    return EXIT_SUCCESS;
}
{'fraction_non_alphanumeric': 0.12342301261589907, 'fraction_numerical': 0.006839945280437756, 'mean_word_length': 3.354731965585705, 'pattern_counts': {'":': 0, '<': 66, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6318076', 'n_tokens_mistral': 2072, 'n_tokens_neox': 1970, 'n_words': 533}