<commit_before>/** \file   SimpleXmlParser.cc
 *  \brief  A non-validating XML parser class.
 *  \author Dr. <NAME> (<EMAIL>)
 *
 *  \copyright 2015 Universitätsbiblothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "SimpleXmlParser.h"
#include <stdexcept>
#include "Compiler.h"
#include "StringUtil.h"
#include "TextUtil.h"


namespace {


bool DecodeEnity(const std::string &entity_string, std::string * const decoded_char) {
    if (unlikely(entity_string.empty()))
        return false;

    if (unlikely(entity_string[0] == '#')) {
        if (entity_string.length() < 2)
            return false;

        unsigned code_point;
        if (entity_string[1] == 'x') {
            if (entity_string.length() < 3 or entity_string.length() > 6)
                return false;
            if (not StringUtil::ToUnsigned(entity_string.substr(2), &code_point, 16))
                return false;
        } else {
            if (entity_string.length() < 2 or entity_string.length() > 6)
                return false;
            if (not StringUtil::ToUnsigned(entity_string.substr(1), &code_point))
                return false;
        }

        if (not TextUtil::WCharToUTF8String(std::wstring(1, static_cast<wchar_t>(code_point)), decoded_char))
            return false;
    } else if (entity_string == "quot")
        *decoded_char = "\"";
    else if (entity_string =="amp")
        *decoded_char = "&";
    else if (entity_string =="apos")
        *decoded_char = "'";
    else if (entity_string =="lt")
        *decoded_char = "<";
    else if (entity_string =="gt")
        *decoded_char = ">";
    else
        return false;

    return true;
}


inline bool DecodeEntities(const std::string &raw_string, std::string * const decoded_string) {
    bool in_entity(false);
    std::string entity;
    for (const auto ch : raw_string) {
        if (unlikely(in_entity)) {
            if (ch == ';') {
                std::string decoded_char;
                if (not DecodeEnity(entity, &decoded_char))
                    return false;
                *decoded_string += decoded_char;
                in_entity = false;
            } else
                entity += ch;
        } else if (unlikely(ch == '&')) {
            in_entity = true;
            entity.clear();
        } else
            *decoded_string += ch;
    }

    return not in_entity;
}


} // unnamed namespace


inline void SimpleXmlParser::skipWhiteSpace() {
    for (;;) {
        const int ch(input_->get());
        if (unlikely(ch == EOF))
            return;
        if (ch != ' ' and ch != '\t' and ch != '\n' and ch != '\r') {
            input_->putback(ch);
            return;
        } else if (ch == '\n')
            ++line_no_;
    }
}


inline bool SimpleXmlParser::extractName(std::string * const name) {
    name->clear();

    int ch(input_->get());
    if (unlikely(ch == EOF or (not StringUtil::IsAsciiLetter(ch) and ch != '_' and ch != ':'))) {
        input_->putback(ch);
        return false;
    }

    *name += static_cast<char>(ch);
    for (;;) {
        ch = input_->get();
        if (unlikely(ch == EOF))
            return false;
        if (not (StringUtil::IsAsciiLetter(ch) or StringUtil::IsDigit(ch) or ch == '_' or ch == ':' or ch == '.')) {
            input_->putback(ch);
            return true;
        }
        *name += static_cast<char>(ch);
    }
}


inline bool SimpleXmlParser::extractQuotedString(const int closing_quote, std::string * const s) {
    s->clear();

    for (;;) {
        const int ch(input_->get());
        if (unlikely(ch == EOF))
            return false;
        if (unlikely(ch == closing_quote))
            return true;
        *s += static_cast<char>(ch);
    }
}


bool SimpleXmlParser::getNext(Type * const type, std::map<std::string, std::string> * const attrib_map,
                              std::string * const data)
{
    if (unlikely(last_type_ == ERROR))
        throw std::runtime_error("in SimpleXmlParser::getNext: previous call already indicated an error!");

    attrib_map->clear();
    data->clear();

    if (last_element_was_empty_) {
        last_type_ = *type = CLOSING_TAG;
        data->swap(last_tag_name_);
        last_element_was_empty_ = false;
        last_type_ = CLOSING_TAG;
        return true;
    }

    int ch;
    if (last_type_ == OPENING_TAG) {
        last_type_ = *type = CHARACTERS;

        std::string raw_string;
        while ((ch = input_->get()) != '<') {
            if (unlikely(ch == EOF)) {
                last_error_message_ = "Unexpected EOF while looking for the start of a closing tag!";
                return false;
            }
            if (unlikely(ch == '\n'))
                ++line_no_;
            raw_string += static_cast<char>(ch);
        }
        input_->putback(ch); // Putting back the '<'.

        if (not DecodeEntities(raw_string, data)) {
            last_type_ = *type = ERROR;
            last_error_message_ = "Invalid entity in character data ending on line " + std::to_string(line_no_) + "!";
            return false;
        }
    } else { // end-of-document or opening or closing tag
        skipWhiteSpace();

        ch = input_->get();
        if (unlikely(ch == EOF)) {
            last_type_ = *type = END_OF_DOCUMENT;
            return true;
        }

        if (ch != '<') {
            last_type_ = *type = ERROR;
            last_error_message_ = "Expected '<' on line " + std::to_string(line_no_) + ", found '"
                                  + std::string(1, static_cast<char>(ch)) + "' instead!";
            return false;<commit_msg>Still faster!<commit_after>/** \file   SimpleXmlParser.cc
 *  \brief  A non-validating XML parser class.
 *  \author Dr. <NAME> (<EMAIL>)
 *
 *  \copyright 2015 Universitätsbiblothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "SimpleXmlParser.h"
#include <stdexcept>
#include "Compiler.h"
#include "StringUtil.h"
#include "TextUtil.h"


namespace {


bool DecodeEnity(const std::string &entity_string, std::string * const decoded_char) {
    if (unlikely(entity_string.empty()))
        return false;

    if (unlikely(entity_string[0] == '#')) {
        if (entity_string.length() < 2)
            return false;

        unsigned code_point;
        if (entity_string[1] == 'x') {
            if (entity_string.length() < 3 or entity_string.length() > 6)
                return false;
            if (not StringUtil::ToUnsigned(entity_string.substr(2), &code_point, 16))
                return false;
        } else {
            if (entity_string.length() < 2 or entity_string.length() > 6)
                return false;
            if (not StringUtil::ToUnsigned(entity_string.substr(1), &code_point))
                return false;
        }

        if (not TextUtil::WCharToUTF8String(std::wstring(1, static_cast<wchar_t>(code_point)), decoded_char))
            return false;
    } else if (entity_string == "quot")
        *decoded_char = "\"";
    else if (entity_string =="amp")
        *decoded_char = "&";
    else if (entity_string =="apos")
        *decoded_char = "'";
    else if (entity_string =="lt")
        *decoded_char = "<";
    else if (entity_string =="gt")
        *decoded_char = ">";
    else
        return false;

    return true;
}

    
// Replaces entities in the range (before,last].
inline bool DecodeEntities(std::string * const data) {
    if (unlikely(data->empty()))
        return true;

    std::string::iterator next(data->begin());
    bool in_entity(false);
    std::string entity;
    for (const auto ch : *data) {
        if (unlikely(in_entity)) {
            if (ch == ';') {
                std::string decoded_char;
                if (not DecodeEnity(entity, &decoded_char))
                    return false;
                for (const auto dch : decoded_char)
                    *next++ = dch;
                in_entity = false;
            } else
                entity += ch;
        } else if (unlikely(ch == '&')) {
            in_entity = true;
            entity.clear();
        } else
            *next++ = ch;
    }

    if (unlikely(next != data->end())) // We had some non-length-preserving entity replacements.
        data->resize(next - data->begin());

    return not in_entity;
}


} // unnamed namespace


inline void SimpleXmlParser::skipWhiteSpace() {
    for (;;) {
        const int ch(input_->get());
        if (unlikely(ch == EOF))
            return;
        if (ch != ' ' and ch != '\t' and ch != '\n' and ch != '\r') {
            input_->putback(ch);
            return;
        } else if (ch == '\n')
            ++line_no_;
    }
}


inline bool SimpleXmlParser::extractName(std::string * const name) {
    name->clear();

    int ch(input_->get());
    if (unlikely(ch == EOF or (not StringUtil::IsAsciiLetter(ch) and ch != '_' and ch != ':'))) {
        input_->putback(ch);
        return false;
    }

    *name += static_cast<char>(ch);
    for (;;) {
        ch = input_->get();
        if (unlikely(ch == EOF))
            return false;
        if (not (StringUtil::IsAsciiLetter(ch) or StringUtil::IsDigit(ch) or ch == '_' or ch == ':' or ch == '.')) {
            input_->putback(ch);
            return true;
        }
        *name += static_cast<char>(ch);
    }
}


inline bool SimpleXmlParser::extractQuotedString(const int closing_quote, std::string * const s) {
    s->clear();

    for (;;) {
        const int ch(input_->get());
        if (unlikely(ch == EOF))
            return false;
        if (unlikely(ch == closing_quote))
            return true;
        *s += static_cast<char>(ch);
    }
}


bool SimpleXmlParser::getNext(Type * const type, std::map<std::string, std::string> * const attrib_map,
                              std::string * const data)
{
    if (unlikely(last_type_ == ERROR))
        throw std::runtime_error("in SimpleXmlParser::getNext: previous call already indicated an error!");

    attrib_map->clear();
    data->clear();

    if (last_element_was_empty_) {
        last_type_ = *type = CLOSING_TAG;
        data->swap(last_tag_name_);
        last_element_was_empty_ = false;
        last_type_ = CLOSING_TAG;
        return true;
    }

    int ch;
    if (last_type_ == OPENING_TAG) {
        last_type_ = *type = CHARACTERS;

        while ((ch = input_->get()) != '<') {
            if (unlikely(ch == EOF)) {
                last_error_message_ = "Unexpected EOF while looking for the start of a closing tag!";
                return false;
            }
            if (unlikely(ch == '\n'))
                ++line_no_;
            *data += static_cast<char>(ch);
        }
        input_->putback(ch); // Putting back the '<'.

        if (not DecodeEntities(data)) {
            last_type_ = *type = ERROR;
            last_error_message_ = "Invalid entity in character data ending on line " + std::to_string(line_no_) + "!";
            return false;
        }
    } else { // end-of-document or opening or closing tag
        skipWhiteSpace();

        ch = input_->get();
        if (unlikely(ch == EOF)) {
            last_type_ = *type = END_OF_DOCUMENT;
            return true;
        }

        if (ch != '<') {
            last_type_ = *type = ERROR;
            last_error_message_ = "Expected '<' on line " + std::to_string(line_no_) + ", found '"
                                  + std::string(1, static_cast<char>(ch)) + "' instead!";
            return false;
{'fraction_non_alphanumeric': 0.1316263409286665, 'fraction_numerical': 0.002975491347584371, 'mean_word_length': 2.1128442602973436, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1184446', 'n_tokens_mistral': 3887, 'n_tokens_neox': 3782, 'n_words': 1030}