<commit_before>/** \brief Utility to determine Zeder entries that have yet to be imported into zts_harvester.
 *  \author <NAME> (<EMAIL>)
 *
 *  \copyright 2019 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <stdexcept>
#include <cstdlib>
#include <algorithm>
#include <set>
#include "IniFile.h"
#include "JournalConfig.h"
#include "MiscUtil.h"
#include "StringUtil.h"
#include "util.h"
#include "Zeder.h"


namespace {


void LoadToolConfig(const IniFile &config_file, const Zeder::Flavour flavour,
                    std::unordered_map<std::string, std::string> * const filter_regexps)
{
    const auto flavour_section(config_file.getSection(Zeder::FLAVOUR_TO_STRING_MAP.at(flavour)));
    for (const auto &column_name : flavour_section->getEntryNames())
        (*filter_regexps)[column_name] = flavour_section->getString(column_name);
}


void DownloadFullDump(const Zeder::Flavour flavour, const std::unordered_map<std::string, std::string> & filter_regexps,
                      Zeder::EntryCollection * const downloaded_entries)
{
    const auto endpoint_url(Zeder::GetFullDumpEndpointPath(flavour));
    std::unique_ptr<Zeder::FullDumpDownloader::Params> downloader_params(new Zeder::FullDumpDownloader::Params(endpoint_url,
                                                                         {}, filter_regexps));

    auto downloader(Zeder::FullDumpDownloader::Factory(Zeder::FullDumpDownloader::Type::FULL_DUMP, std::move(downloader_params)));
    if (not downloader->download(downloaded_entries))
        LOG_ERROR("Couldn't download full dump for " + Zeder::FLAVOUR_TO_STRING_MAP.at(flavour));
}
<commit_msg>zeder_count_imported: fix syntax for CentOS compatibility<commit_after>/** \brief Utility to determine Zeder entries that have yet to be imported into zts_harvester.
 *  \author <NAME> (<EMAIL>)
 *
 *  \copyright 2019 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <stdexcept>
#include <cstdlib>
#include <algorithm>
#include <set>
#include "IniFile.h"
#include "JournalConfig.h"
#include "MiscUtil.h"
#include "StringUtil.h"
#include "util.h"
#include "Zeder.h"


namespace {


void LoadToolConfig(const IniFile &config_file, const Zeder::Flavour flavour,
                    std::unordered_map<std::string, std::string> * const filter_regexps)
{
    const auto flavour_section(config_file.getSection(Zeder::FLAVOUR_TO_STRING_MAP.at(flavour)));
    for (const auto &column_name : flavour_section->getEntryNames())
        (*filter_regexps)[column_name] = flavour_section->getString(column_name);
}


void DownloadFullDump(const Zeder::Flavour flavour, const std::unordered_map<std::string, std::string> & filter_regexps,
                      Zeder::EntryCollection * const downloaded_entries)
{
    const auto endpoint_url(Zeder::GetFullDumpEndpointPath(flavour));
    const std::unordered_set<std::string> columns_to_download;  // intentionally empty
    std::unique_ptr<Zeder::FullDumpDownloader::Params> downloader_params(new Zeder::FullDumpDownloader::Params(endpoint_url,
                                                                         columns_to_download, filter_regexps));

    auto downloader(Zeder::FullDumpDownloader::Factory(Zeder::FullDumpDownloader::Type::FULL_DUMP, std::move(downloader_params)));
    if (not downloader->download(downloaded_entries))
        LOG_ERROR("Couldn't download full dump for " + Zeder::FLAVOUR_TO_STRING_MAP.at(flavour));
}

{'fraction_non_alphanumeric': 0.09816971713810316, 'fraction_numerical': 0.0020798668885191347, 'mean_word_length': 4.176533907427341, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5366278', 'n_tokens_mistral': 1432, 'n_tokens_neox': 1372, 'n_words': 409}