<commit_before>/** \brief A tool for rewriting  information in ssoar data
 *  \author <NAME>
 *
 *  \copyright 2018 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <iostream>
#include <memory>
#include <unordered_map>
#include <unordered_set>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include "Compiler.h"
#include "MARC.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "util.h"


namespace {


[[noreturn]] void Usage() {
    std::cerr << "Usage: " << ::progname << " marc_input marc_output\n";
    std::exit(EXIT_FAILURE);
}


void Assemble773Article(MARC::Subfields * const _773subfields, const std::string &title = "",
                        const std::string &year = "", const std::string &pages = "",
                        const std::string &volinfo = "", const std::string &edition = "")
{
    if (not (title.empty() and volinfo.empty() and pages.empty() and year.empty() and edition.empty()))
       _773subfields->addSubfield('i', "In:");
    if (not title.empty())
        _773subfields->addSubfield('a', StringUtil::Trim(title));
    if (not volinfo.empty())
        _773subfields->addSubfield('g', "volume: " + StringUtil::Trim(volinfo));
    if (not pages.empty())
        _773subfields->addSubfield('g', "pages: " + pages);
    if (not year.empty())
        _773subfields->addSubfield('g', "year: " + year);
    if (not edition.empty())
        _773subfields->addSubfield('g', "edition: "  + edition);
}


void Assemble773Book(MARC::Subfields * const _773subfields, const std::string &title = "",
                     const std::string &authors = "", const std::string &year = "",
                     const std::string &pages = "", const std::string &isbn = "")
{
    if (not (title.empty() and authors.empty() and year.empty() and pages.empty() and isbn.empty()))
       _773subfields->addSubfield('i', "In:");
    if (not title.empty()) {
        if (not authors.empty())
            _773subfields->addSubfield('t', StringUtil::Trim(title));
        else
            _773subfields->addSubfield('a', StringUtil::Trim(title));
    }
    if (not authors.empty())
        _773subfields->addSubfield('a', authors);
    if (not year.empty())
        _773subfields->addSubfield('d', year);
    if ( not pages.empty())
        _773subfields->addSubfield('g', "pages:" + pages);
    if (not isbn.empty())
        _773subfields->addSubfield('o', isbn);
}


void Assemble936Article(MARC::Subfields * const _936subfields,
                        const std::string &year = "", const std::string &pages = "",
                        const std::string &volinfo = "", const std::string &edition = "") {

    if (volinfo.empty() and pages.empty() and year.empty() and edition.empty() and volinfo.empty())
        return;
    if (not volinfo.empty())
        _936subfields->addSubfield('d', StringUtil::Trim(volinfo));
    if (not year.empty())
        _936subfields->addSubfield('j', year);
    if (not pages.empty())
        _936subfields->addSubfield('h', pages);
    if (not edition.empty())
        _936subfields->addSubfield('e', edition);<commit_msg>Extract additional information from volinfo<commit_after>/** \brief A tool for rewriting  information in ssoar data
 *  \author <NAME>
 *
 *  \copyright 2018 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <iostream>
#include <memory>
#include <unordered_map>
#include <unordered_set>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include "Compiler.h"
#include "MARC.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "util.h"


namespace {


[[noreturn]] void Usage() {
    std::cerr << "Usage: " << ::progname << " marc_input marc_output\n";
    std::exit(EXIT_FAILURE);
}


void Assemble773Article(MARC::Subfields * const _773subfields, const std::string &title = "",
                        const std::string &year = "", const std::string &pages = "",
                        const std::string &volinfo = "", const std::string &edition = "")
{
    if (not (title.empty() and volinfo.empty() and pages.empty() and year.empty() and edition.empty()))
       _773subfields->addSubfield('i', "In:");
    if (not title.empty())
        _773subfields->addSubfield('a', StringUtil::Trim(title));
    if (not volinfo.empty())
        _773subfields->addSubfield('g', "volume: " + StringUtil::Trim(volinfo));
    if (not pages.empty())
        _773subfields->addSubfield('g', "pages: " + pages);
    if (not year.empty())
        _773subfields->addSubfield('g', "year: " + year);
    if (not edition.empty())
        _773subfields->addSubfield('g', "edition: "  + edition);
}


void Assemble773Book(MARC::Subfields * const _773subfields, const std::string &title = "",
                     const std::string &authors = "", const std::string &year = "",
                     const std::string &pages = "", const std::string &isbn = "")
{
    if (not (title.empty() and authors.empty() and year.empty() and pages.empty() and isbn.empty()))
       _773subfields->addSubfield('i', "In:");
    if (not title.empty()) {
        if (not authors.empty())
            _773subfields->addSubfield('t', StringUtil::Trim(title));
        else
            _773subfields->addSubfield('a', StringUtil::Trim(title));
    }
    if (not authors.empty())
        _773subfields->addSubfield('a', authors);
    if (not year.empty())
        _773subfields->addSubfield('d', year);
    if ( not pages.empty())
        _773subfields->addSubfield('g', "pages:" + pages);
    if (not isbn.empty())
        _773subfields->addSubfield('o', isbn);
}


bool ParseVolInfo(const std::string &volinfo, std::string * const volinfo_vol, std::string * const volinfo_year,
                  std::string * const volinfo_edition) {
    static const std::string volinfo_regex("(\\d+)\\s+\\((\\d{4})\\)\\s+(\\d+)"); //vol (year) edition
    static RegexMatcher * const vol_info_matcher(RegexMatcher::RegexMatcherFactoryOrDie(volinfo_regex));
    if (not vol_info_matcher->matched(volinfo))
        return false;

    *volinfo_vol = (*vol_info_matcher)[1];
    *volinfo_year = (*vol_info_matcher)[2];
    *volinfo_edition = (*vol_info_matcher)[3];
    return true;
}


void Assemble936Article(MARC::Subfields * const _936subfields,
                        const std::string &year = "", const std::string &pages = "",
                        const std::string &volinfo = "", const std::string &edition = "") {

    if (volinfo.empty() and pages.empty() and year.empty() and edition.empty() and volinfo.empty())
        return;

    std::string volinfo_vol;
    std::string volinfo_year;
    std::string volinfo_edition;
    // Volinfo might also contain vol year edition in the format "vol (year) edition"
    if (not volinfo.empty()) {
        if (ParseVolInfo(volinfo, &volinfo_vol, &volinfo_year, &volinfo_edition))
            _936subfields->addSubfield('d', volinfo_vol);
        else
            _936subfields->addSubfield('d', StringUtil::Trim(volinfo));
    }
    if (not (year.empty() and volinfo_year.empty()))
        _936subfields->addSubfield('j', (not year.empty()) ? year : volinfo_year);
    if (not pages.empty())
        _936subfields->addSubfield('h', pages);
    if (not (edition.empty() and volinfo_edition.empty()))
        _936subfields->addSubfield('e', (not edition.empty()) ? edition : volinfo_edition);
{'fraction_non_alphanumeric': 0.14673725399930948, 'fraction_numerical': 0.017838646564621936, 'mean_word_length': 3.2786804529788283, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5790425', 'n_tokens_mistral': 2784, 'n_tokens_neox': 2636, 'n_words': 731}