<commit_before>/** \brief Interaction with Zotero Translation Server
 *         public functions are named like endpoints
 *         see https://github.com/zotero/translation-server
 *  \author Dr. <NAME>
 *  \author <NAME>
 *
 *  \copyright 2018 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
#include "Zotero.h"
#include <ctime>
#include <kchashdb.h>
#include <uuid/uuid.h>
#include "DbConnection.h"
#include "MiscUtil.h"
#include "SqlUtil.h"
#include "StringUtil.h"
#include "SyndicationFormat.h"
#include "UrlUtil.h"
#include "WebUtil.h"
#include "util.h"


// Forward declaration:
class File;


namespace Zotero {


const std::string DEFAULT_SIMPLE_CRAWLER_CONFIG_PATH("/usr/local/var/lib/tuelib/zotero_crawler.conf");
const std::string ISSN_TO_PPN_MAP_PATH("/usr/local/var/lib/tuelib/issn_to_ppn.map");


const std::vector<std::string> EXPORT_FORMATS{
    "bibtex", "biblatex", "bookmarks", "coins", "csljson", "mods", "refer",
    "rdf_bibliontology", "rdf_dc", "rdf_zotero", "ris", "wikipedia", "tei",
    "json", "marc21", "marcxml"
};


const std::string DEFAULT_SUBFIELD_CODE("eng");


namespace {


struct Date {
    static const unsigned INVALID = 0;
    unsigned day_;
    unsigned month_;
    unsigned year_;
public:
    Date(): day_(INVALID), month_(INVALID), year_(INVALID) { }
};


Date StringToDate(const std::string &date_str, const std::string &optional_strptime_format) {
    Date date;

    time_t unix_time(TimeUtil::BAD_TIME_T);
    if (optional_strptime_format.empty())
        unix_time = WebUtil::ParseWebDateAndTime(date_str);
    else {
        struct tm tm;
        std::vector<std::string> format_string_splits;

        // try available format strings until a matching one is found
        if (StringUtil::SplitThenTrimWhite(optional_strptime_format, '|', &format_string_splits)) {
            for (const auto &format_string : format_string_splits) {
                std::memset(&tm, 0, sizeof(tm));
                const char * const last_char(::strptime(date_str.c_str(), format_string.c_str(), &tm));
                if (last_char == nullptr or *last_char != '\0')
                    unix_time = TimeUtil::BAD_TIME_T;
                else {
                    date.year_ = tm.tm_year + 1900;
                    date.month_ = tm.tm_mon + 1;
                    date.day_ = tm.tm_mday;
                    if (date.day_ == 0)<commit_msg>Added support for optional locale specifications.<commit_after>/** \brief Interaction with Zotero Translation Server
 *         public functions are named like endpoints
 *         see https://github.com/zotero/translation-server
 *  \author Dr. <NAME>
 *  \author <NAME>
 *
 *  \copyright 2018 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
#include "Zotero.h"
#include <ctime>
#include <kchashdb.h>
#include <uuid/uuid.h>
#include "DbConnection.h"
#include "Locale.h"
#include "MiscUtil.h"
#include "SqlUtil.h"
#include "StringUtil.h"
#include "SyndicationFormat.h"
#include "UrlUtil.h"
#include "WebUtil.h"
#include "util.h"


// Forward declaration:
class File;


namespace Zotero {


const std::string DEFAULT_SIMPLE_CRAWLER_CONFIG_PATH("/usr/local/var/lib/tuelib/zotero_crawler.conf");
const std::string ISSN_TO_PPN_MAP_PATH("/usr/local/var/lib/tuelib/issn_to_ppn.map");


const std::vector<std::string> EXPORT_FORMATS{
    "bibtex", "biblatex", "bookmarks", "coins", "csljson", "mods", "refer",
    "rdf_bibliontology", "rdf_dc", "rdf_zotero", "ris", "wikipedia", "tei",
    "json", "marc21", "marcxml"
};


const std::string DEFAULT_SUBFIELD_CODE("eng");


namespace {


struct Date {
    static const unsigned INVALID = 0;
    unsigned day_;
    unsigned month_;
    unsigned year_;
public:
    Date(): day_(INVALID), month_(INVALID), year_(INVALID) { }
};


Date StringToDate(const std::string &date_str, std::string optional_strptime_format) {
    Date date;

    time_t unix_time(TimeUtil::BAD_TIME_T);
    if (optional_strptime_format.empty())
        unix_time = WebUtil::ParseWebDateAndTime(date_str);
    else {
        std::unique_ptr<Locale> locale;
        // Optional locale specification?
        if (optional_strptime_format[0] == '(') {
            const size_t closing_paren_pos(optional_strptime_format.find(')', 1));
            if (unlikely(closing_paren_pos == std::string::npos or closing_paren_pos == 1))
                LOG_ERROR("bad local specification \"" + optional_strptime_format + "\"!");
            const std::string locale_specification(optional_strptime_format.substr(1, closing_paren_pos - 1));
            locale.reset(new Locale(locale_specification, LC_TIME));
        }
        
        struct tm tm;
        std::vector<std::string> format_string_splits;

        // try available format strings until a matching one is found
        if (StringUtil::SplitThenTrimWhite(optional_strptime_format, '|', &format_string_splits)) {
            for (const auto &format_string : format_string_splits) {
                std::memset(&tm, 0, sizeof(tm));
                const char * const last_char(::strptime(date_str.c_str(), format_string.c_str(), &tm));
                if (last_char == nullptr or *last_char != '\0')
                    unix_time = TimeUtil::BAD_TIME_T;
                else {
                    date.year_ = tm.tm_year + 1900;
                    date.month_ = tm.tm_mon + 1;
                    date.day_ = tm.tm_mday;
                    if (date.day_ == 0)
{'fraction_non_alphanumeric': 0.11509517485613104, 'fraction_numerical': 0.005459642909842113, 'mean_word_length': 3.2177971375233354, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '771041', 'n_tokens_mistral': 2193, 'n_tokens_neox': 2111, 'n_words': 582}