<commit_before>/** \file   RegexMatcher.cc
 *  \brief  Implementation of the MARC-21 Leader class.
 *  \author Dr. <NAME> (<EMAIL>)
 *
 *  \copyright 2015 Universitätsbiblothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "Leader.h"
#include <cstdio>
#include "StringUtil.h"


const size_t Leader::LEADER_LENGTH(24);


bool Leader::ParseLeader(const std::string &leader_string, std::shared_ptr<Leader> &leader,
			 std::string * const err_msg)
{
    if (err_msg != nullptr)
        err_msg->clear();

    if (leader_string.size() != LEADER_LENGTH) {
        if (err_msg != nullptr)
            *err_msg = "Leader length must be " + std::to_string(LEADER_LENGTH) +
                ", found " + std::to_string(leader_string.size()) + "!";
        return false;
    }

    unsigned record_length;
    if (std::sscanf(leader_string.substr(0, 5).data(), "%5u", &record_length) != 1) {
        if (err_msg != nullptr)
            *err_msg = "Can't parse record length!";
        return false;<commit_msg>When we have a corrupted record length in the leader of a MARC record, display the bytes that were found.<commit_after>/** \file   RegexMatcher.cc
 *  \brief  Implementation of the MARC-21 Leader class.
 *  \author Dr. <NAME> (<EMAIL>)
 *
 *  \copyright 2015 Universitätsbiblothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "Leader.h"
#include <cctype>
#include <cstdio>
#include "StringUtil.h"


const size_t Leader::LEADER_LENGTH(24);


namespace {


std::string EscapeString(const std::string &s) {
    std::string escaped_string;
    escaped_string.reserve(s.length() * 3);

    for (const char ch : s) {
	if (std::isprint(ch))
	    escaped_string += ch;
	else {
	    escaped_string += '\\';
	    escaped_string += StringUtil::ToHex(static_cast<unsigned char>(ch) >> 4);
	    escaped_string += StringUtil::ToHex(static_cast<unsigned char>(ch) & 0xF);
	}
    }

    return escaped_string;
}


}


bool Leader::ParseLeader(const std::string &leader_string, std::shared_ptr<Leader> &leader,
			 std::string * const err_msg)
{
    if (err_msg != nullptr)
        err_msg->clear();

    if (leader_string.size() != LEADER_LENGTH) {
        if (err_msg != nullptr)
            *err_msg = "Leader length must be " + std::to_string(LEADER_LENGTH) +
                ", found " + std::to_string(leader_string.size()) + "!";
        return false;
    }

    unsigned record_length;
    if (std::sscanf(leader_string.substr(0, 5).data(), "%5u", &record_length) != 1) {
        if (err_msg != nullptr)
            *err_msg = "Can't parse record length! (Found \"" + EscapeString(leader_string.substr(0, 5)) + "\"";
        return false;
{'fraction_non_alphanumeric': 0.11412089290193128, 'fraction_numerical': 0.007775269626285428, 'mean_word_length': 3.320693391115926, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6154097', 'n_tokens_mistral': 1276, 'n_tokens_neox': 1210, 'n_words': 411}