<commit_before>/** \file   generate_subscription_packets.cc
 *  \brief  Imports data from Zeder and writes a subscription packets defintion file.
 *  \author Dr. <NAME> (<EMAIL>)
 *  \copyright 2020 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <map>
#include <unordered_map>
#include <set>
#include "Compiler.h"
#include "FileUtil.h"
#include "IniFile.h"
#include "StringUtil.h"
#include "util.h"
#include "Zeder.h"


namespace {


// Return true if an entry of "class_list" equals one of the vertical-bar-separated values of "expected_values_str."
bool FoundExpectedClassValue(const std::string &expected_values_str, const std::string &class_list_str) {
    std::vector<std::string> expected_values;
    StringUtil::Split(expected_values_str, '|', &expected_values);

    std::vector<std::string> class_list;
    StringUtil::SplitThenTrimWhite(class_list_str, ',', &class_list);

    for (const auto &class_str : class_list) {
        if (std::find_if(expected_values.cbegin(), expected_values.cend(),
                         [&class_str](auto &expected_value) { return class_str == expected_value; }) != expected_values.cend())
            return true;
    }
    return false;
}


bool IncludeJournal(const Zeder::Entry &journal, const IniFile::Section &filter_section) {
    for (const auto &entry : filter_section) {
        if (entry.name_.empty() or entry.name_ == "description")
            continue;

        std::string zeder_column_name(entry.name_);
        if (entry.name_ == "except_class")
            zeder_column_name = "class";

        const auto column_value(StringUtil::TrimWhite(journal.lookup(zeder_column_name)));
        if (column_value.empty())
            return false;

        if (zeder_column_name != "class") {
            if (::strcasecmp(column_value.c_str(), entry.value_.c_str()) != 0)
                return false;
        } else { // class or except_class
            const bool found_it(FoundExpectedClassValue(entry.value_, column_value));
            if ((not found_it and entry.name_ == "class")
                or (found_it and entry.name_ == "except_class"))
                return false;
        }
    }

    return true;
}


// Please note that Zeder PPN entries are separated by spaces and, unlike what the column names "print_ppn" and
// "online_ppn" imply may in rare cases contain space-separated lists of PPN's.
void ProcessPPNs(const std::string &ppns, std::set<std::string> * const bundle_ppns) {
    std::vector<std::string> individual_ppns;
    StringUtil::Split(ppns, ' ', &individual_ppns);
    bundle_ppns->insert(individual_ppns.cbegin(), individual_ppns.cend());
}


std::string EscapeDoubleQuotes(const std::string &s) {
    std::string escaped_s;
    escaped_s.reserve(s.size());

    for (const char ch : s) {
        if (ch == '"' or ch == '\\')
            escaped_s += '\\';
        escaped_s += ch;
    }

    return escaped_s;
}


void GenerateBundleDefinition(const Zeder::SimpleZeder &zeder, const std::string &bundle_instances,
                              const IniFile::Section &section, File * const output_file)
{
    unsigned included_journal_count(0);
    std::set<std::string> bundle_ppns; // We use a std::set because it is automatically being sorted for us.
    for (const auto &journal : zeder) {
    if (journal.size() == 1 /* only the column ID */or not IncludeJournal(journal, section))
        continue;

        ++included_journal_count;
        const auto print_ppns(journal.lookup("pppn"));
        const auto online_ppns(journal.lookup("eppn"));

        if (print_ppns.empty() and online_ppns.empty()) {
            --included_journal_count;
            const auto zeder_id(journal.lookup("DT_RowId"));
            LOG_WARNING("Zeder entry #" + zeder_id + " is missing print and online PPN's!");
            continue;
        }
<commit_msg>If at first you don't succeed...<commit_after>/** \file   generate_subscription_packets.cc
 *  \brief  Imports data from Zeder and writes a subscription packets defintion file.
 *  \author Dr. <NAME> (<EMAIL>)
 *  \copyright 2020 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <map>
#include <unordered_map>
#include <set>
#include "Compiler.h"
#include "FileUtil.h"
#include "IniFile.h"
#include "StringUtil.h"
#include "util.h"
#include "Zeder.h"


namespace {


// Return true if an entry of "class_list" equals one of the vertical-bar-separated values of "expected_values_str."
bool FoundExpectedClassValue(const std::string &expected_values_str, const std::string &class_list_str) {
    std::vector<std::string> expected_values;
    StringUtil::Split(expected_values_str, '|', &expected_values);

    std::vector<std::string> class_list;
    StringUtil::SplitThenTrimWhite(class_list_str, ',', &class_list);

    for (const auto &class_str : class_list) {
        if (std::find_if(expected_values.cbegin(), expected_values.cend(),
                         [&class_str](auto &expected_value) { return class_str == expected_value; }) != expected_values.cend())
            return true;
    }
    return false;
}


bool IncludeJournal(const Zeder::Entry &journal, const IniFile::Section &filter_section) {
    for (const auto &entry : filter_section) {
        if (entry.name_.empty() or entry.name_ == "description")
            continue;

        std::string zeder_column_name(entry.name_);
        if (entry.name_ == "except_class")
            zeder_column_name = "class";

        const auto column_value(StringUtil::TrimWhite(journal.lookup(zeder_column_name)));
        if (column_value.empty())
            return false;

        if (zeder_column_name != "class") {
            if (::strcasecmp(column_value.c_str(), entry.value_.c_str()) != 0)
                return false;
        } else { // class or except_class
            const bool found_it(FoundExpectedClassValue(entry.value_, column_value));
            if ((not found_it and entry.name_ == "class")
                or (found_it and entry.name_ == "except_class"))
                return false;
        }
    }

    return true;
}


// Please note that Zeder PPN entries are separated by spaces and, unlike what the column names "print_ppn" and
// "online_ppn" imply may in rare cases contain space-separated lists of PPN's.
void ProcessPPNs(const std::string &ppns, std::set<std::string> * const bundle_ppns) {
    std::vector<std::string> individual_ppns;
    StringUtil::Split(ppns, ' ', &individual_ppns);
    bundle_ppns->insert(individual_ppns.cbegin(), individual_ppns.cend());
}


std::string EscapeDoubleQuotes(const std::string &s) {
    std::string escaped_s;
    escaped_s.reserve(s.size());

    for (const char ch : s) {
        if (ch == '"' or ch == '\\')
            escaped_s += '\\';
        escaped_s += ch;
    }

    return escaped_s;
}


void GenerateBundleDefinition(const Zeder::SimpleZeder &zeder, const std::string &bundle_instances,
                              const IniFile::Section &section, File * const output_file)
{
    unsigned included_journal_count(0);
    std::set<std::string> bundle_ppns; // We use a std::set because it is automatically being sorted for us.
    for (const auto &journal : zeder) {
        if (journal.size() == 1 /* only the column ID */or not IncludeJournal(journal, section))
            continue;

        ++included_journal_count;
        const auto print_ppns(journal.lookup("pppn"));
        const auto online_ppns(journal.lookup("eppn"));

        if (print_ppns.empty() and online_ppns.empty()) {
            --included_journal_count;
            const auto zeder_id(journal.lookup("DT_RowId"));
            LOG_WARNING("Zeder entry #" + zeder_id + " is missing print and online PPN's!");
            continue;
        }

{'fraction_non_alphanumeric': 0.11344399252500824, 'fraction_numerical': 0.0017588215895350116, 'mean_word_length': 3.1714809720311785, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1349061', 'n_tokens_mistral': 2737, 'n_tokens_neox': 2657, 'n_words': 791}