<commit_before>// Copyright (C) 2019 <NAME> <<EMAIL>>
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

#include <primitives/xml.h>

#include <primitives/exceptions.h>

#include <boost/algorithm/string.hpp>

#include <iomanip>
#include <sstream>

void checkNullptr(const void *from)
{
    if (!from)
        throw SW_RUNTIME_ERROR("nullptr");
}

String getName(xmlNode *from)
{
    checkNullptr(from);
    checkNullptr(from->name);
    return (char *)from->name;
}

String getName(xmlAttr *from)
{
    checkNullptr(from);
    checkNullptr(from->name);
    return (char *)from->name;
}

String getContent(xmlNode *from)
{
    checkNullptr(from);
    if (from->type == XML_ELEMENT_NODE)
    {
        from = &from->children[0];
        return getContent(from);
    }
    if (from->type == XML_TEXT_NODE)
        return (char*)from->content;
    throw SW_RUNTIME_ERROR("unknown node type");
}

static xmlNode *getNextRaw(xmlNode *from, const String &name)
{
    if (!from)
        return nullptr;

    for (auto cur_node = from; cur_node; cur_node = cur_node->next)
    {
        if (cur_node->type == XML_ELEMENT_NODE)
        {
            if (getName(cur_node) == name)
                return cur_node;
        }
        auto n = getNextRaw(cur_node->children, name);
        if (n)
            return n;
    }
    return nullptr;
}

xmlNode *getNext(xmlNode *from, const String &name, int skip)
{
    while (skip--)
        from = getNext(from, name);
    checkNullptr(from);
    if (getName(from) == name)
    {
        if (from->children)
        {
            auto n = getNextRaw(from->children, name);
            if (n)
                return n;
        }
        if (from->next)
        {
            auto n = getNextRaw(from->next, name);
            if (n)
                return n;
        }
        return nullptr;
    }
    auto n = getNextRaw(from, name);
    if (n)
        return n;
    throw SW_RUNTIME_ERROR(name + ": not found");
}

xmlNode *findNodeWithText(xmlNode *from, const String &text)
{
    if (!from)
        return nullptr;

    for (auto cur_node = from; cur_node; cur_node = cur_node->next)
    {
        //if (cur_node->type == XML_ELEMENT_NODE)
        {
            if (cur_node->content && String((char*)cur_node->content).find(text) != String::npos)
                return cur_node;
        }
        auto n = findNodeWithText(cur_node->children, text);
        if (n)
            return n;
    }
    return nullptr;
}

void removeNode(xmlNode *n)
{
    xmlUnlinkNode(n);
    xmlFreeNode(n);
}

void removeTags(xmlNode *from, const String &tag)
{
    while (auto n = getNextRaw(from, tag))
        removeNode(n);
}

void removeComments(xmlNode *from)
{
    if (!from)
        return;

    if (from->content)
    {
        String s = (const char *)from->content;
        boost::trim(s);
        memcpy(from->content, s.data(), s.size() + 1);
    }

    for (auto cur_node = from; cur_node;)
    {
        removeComments(cur_node->children);

        auto n = cur_node;
        cur_node = cur_node->next;

        if (n->type == XML_COMMENT_NODE)
            removeNode(n);
    }
}

String convertTimestamp(const String &s)
{
    std::tm t = {};
    std::istringstream ss(s);
    ss >> std::get_time(&t, "%d.%m.%Y %H:%M:%S");
    if (ss.fail())
        throw SW_RUNTIME_ERROR("Date parse failed");
    char mbstr[100];
    if (std::strftime(mbstr, sizeof(mbstr), "%F %T", &t)) {
        return mbstr;
    }
    throw SW_RUNTIME_ERROR("strftime error");
}
<commit_msg>[xml] Return empty content on empty child nodes.<commit_after>// Copyright (C) 2019 <NAME> <<EMAIL>>
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

#include <primitives/xml.h>

#include <primitives/exceptions.h>

#include <boost/algorithm/string.hpp>

#include <iomanip>
#include <sstream>

void checkNullptr(const void *from)
{
    if (!from)
        throw SW_RUNTIME_ERROR("nullptr");
}

String getName(xmlNode *from)
{
    checkNullptr(from);
    checkNullptr(from->name);
    return (char *)from->name;
}

String getName(xmlAttr *from)
{
    checkNullptr(from);
    checkNullptr(from->name);
    return (char *)from->name;
}

String getContent(xmlNode *from)
{
    checkNullptr(from);
    if (from->type == XML_ELEMENT_NODE)
    {
        from = &from->children[0];
        if (!from)
            return {};
        return getContent(from);
    }
    if (from->type == XML_TEXT_NODE)
        return (char*)from->content;
    throw SW_RUNTIME_ERROR("unknown node type");
}

static xmlNode *getNextRaw(xmlNode *from, const String &name)
{
    if (!from)
        return nullptr;

    for (auto cur_node = from; cur_node; cur_node = cur_node->next)
    {
        if (cur_node->type == XML_ELEMENT_NODE)
        {
            if (getName(cur_node) == name)
                return cur_node;
        }
        auto n = getNextRaw(cur_node->children, name);
        if (n)
            return n;
    }
    return nullptr;
}

xmlNode *getNext(xmlNode *from, const String &name, int skip)
{
    while (skip--)
        from = getNext(from, name);
    checkNullptr(from);
    if (getName(from) == name)
    {
        if (from->children)
        {
            auto n = getNextRaw(from->children, name);
            if (n)
                return n;
        }
        if (from->next)
        {
            auto n = getNextRaw(from->next, name);
            if (n)
                return n;
        }
        return nullptr;
    }
    auto n = getNextRaw(from, name);
    if (n)
        return n;
    throw SW_RUNTIME_ERROR(name + ": not found");
}

xmlNode *findNodeWithText(xmlNode *from, const String &text)
{
    if (!from)
        return nullptr;

    for (auto cur_node = from; cur_node; cur_node = cur_node->next)
    {
        //if (cur_node->type == XML_ELEMENT_NODE)
        {
            if (cur_node->content && String((char*)cur_node->content).find(text) != String::npos)
                return cur_node;
        }
        auto n = findNodeWithText(cur_node->children, text);
        if (n)
            return n;
    }
    return nullptr;
}

void removeNode(xmlNode *n)
{
    xmlUnlinkNode(n);
    xmlFreeNode(n);
}

void removeTags(xmlNode *from, const String &tag)
{
    while (auto n = getNextRaw(from, tag))
        removeNode(n);
}

void removeComments(xmlNode *from)
{
    if (!from)
        return;

    if (from->content)
    {
        String s = (const char *)from->content;
        boost::trim(s);
        memcpy(from->content, s.data(), s.size() + 1);
    }

    for (auto cur_node = from; cur_node;)
    {
        removeComments(cur_node->children);

        auto n = cur_node;
        cur_node = cur_node->next;

        if (n->type == XML_COMMENT_NODE)
            removeNode(n);
    }
}

String convertTimestamp(const String &s)
{
    std::tm t = {};
    std::istringstream ss(s);
    ss >> std::get_time(&t, "%d.%m.%Y %H:%M:%S");
    if (ss.fail())
        throw SW_RUNTIME_ERROR("Date parse failed");
    char mbstr[100];
    if (std::strftime(mbstr, sizeof(mbstr), "%F %T", &t)) {
        return mbstr;
    }
    throw SW_RUNTIME_ERROR("strftime error");
}

{'fraction_non_alphanumeric': 0.13837240635947184, 'fraction_numerical': 0.0035030988951765024, 'mean_word_length': 2.227391304347826, 'pattern_counts': {'":': 2, '<': 19, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 31}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4697339', 'n_tokens_mistral': 2521, 'n_tokens_neox': 2464, 'n_words': 606}