<commit_before>#include <node.h>
#include <v8.h>
#include <sstream>
#include <string.h>
#include "rapidxml.hpp"

// Used example code from:
// https://github.com/glynos/cpp-netlib/blob/master/contrib/http_examples/rss/rss.cpp
// http://nodejs.org/api/addons.html

using namespace v8;
using namespace rapidxml;

char const *EMPTY_C_STRING = "";

// Helper to read text node value.
// Returns 0 when cannot read the value.

char const *readTextNode(xml_node<char> *rootNode, const char* name) {
    xml_node<char> *node = rootNode->first_node(name);
    if (node) {
        xml_node<char> *textNode = node->first_node();
        if (textNode) {
            return textNode->value();
        } else {
            return EMPTY_C_STRING;
        }
    } else {
        return 0;
    }
}

// Helper to find the line number of error.

std::pair<int, int> findErrorLine(const char* xml, const char* where) {
    int i = 0;
    int ln = 1;
    char ch = 0;
    int col = 1;
    while (true) {
        if (xml + i == where) {
            break;
        }
        ch = xml[i];
        if (ch == '\n') {
            ln++;
            col = 0;
        } else {
            col++;
        }
        i++;
    }
    return std::pair<int, int>(ln, col);
}

// Parses the Atom feed.

void parseAtomFeed(xml_node<char> *feedNode, const Local<Object> &feed, bool extractContent) {

    feed->Set(String::NewSymbol("type"), String::New("atom"));

    // Extracts the title property.

    char const *title = readTextNode(feedNode, "title");
    if (title) {
        feed->Set(String::NewSymbol("title"), String::New(title));
    }

    // Extracts the id property.

    char const *id = readTextNode(feedNode, "id");
    if (id) {
        feed->Set(String::NewSymbol("id"), String::New(id));
    }

    // Extracts the link property.
    xml_node<char> *linkNode = feedNode->first_node("link");
    if (linkNode) {
        xml_attribute<char> *hrefAttr = linkNode->first_attribute("href");
        if (hrefAttr) {
            feed->Set(String::NewSymbol("link"), String::New(hrefAttr->value()));
        }
    }

    // Extracts the author property.

    char const *author = readTextNode(feedNode, "author");
    if (author) {
        feed->Set(String::NewSymbol("author"), String::New(author));
    }

    // Extract all channel items.

    Local<Array> items = Array::New();
    xml_node<char> *itemNode = feedNode->first_node("entry");

    int i = 0;
    while (itemNode) {
        Local<Object> item = Object::New();

        // Extracts the id property.

        char const *id = readTextNode(itemNode, "id");
        if (id) {
            item->Set(String::NewSymbol("id"), String::New(id));
        }

        Local<Array> links = Array::New();

        // Extracts all links.
        // 4.2.7. The "atom:link" Element

        xml_node<char> *linkNode = itemNode->first_node("link");

        int linkIndex = 0;

        while (linkNode) {

            Local<Object> link = Object::New();

            xml_attribute<char> *relAttr = linkNode->first_attribute("rel");
            xml_attribute<char> *hrefAttr = linkNode->first_attribute("href");
            xml_attribute<char> *typeAttr = linkNode->first_attribute("type");
            xml_attribute<char> *hreflangAttr = linkNode->first_attribute("hreflang");
            xml_attribute<char> *titleAttr = linkNode->first_attribute("title");
            xml_attribute<char> *lengthAttr = linkNode->first_attribute("length");

            if (relAttr) {
                link->Set(String::NewSymbol("rel"), String::New(relAttr->value()));
            }

            if (hrefAttr) {
                link->Set(String::NewSymbol("href"), String::New(hrefAttr->value()));
            }

            if (typeAttr) {
                link->Set(String::NewSymbol("type"), String::New(typeAttr->value()));
            }

            if (hreflangAttr) {
                link->Set(String::NewSymbol("hreflang"), String::New(hreflangAttr->value()));
            }

            if (titleAttr) {
                link->Set(String::NewSymbol("title"), String::New(titleAttr->value()));
            }

            if (lengthAttr) {
                link->Set(String::NewSymbol("length"), String::New(lengthAttr->value()));
            }

            xml_node<char> *textNode = linkNode->first_node();

            if (textNode) {
                link->Set(String::NewSymbol("text"), String::New(textNode->value()));
            }

            links->Set(Number::New(linkIndex), link);

            linkIndex++;

            linkNode = linkNode->next_sibling("link");
        }

        item->Set(String::NewSymbol("links"), links);

        // Extract the item title.

        char const *title = readTextNode(itemNode, "title");
        if (title) {
            item->Set(String::NewSymbol("title"), String::New(title));
        }

        // Extract the published property.
<commit_msg>Comment the usage of Atom link text node<commit_after>#include <node.h>
#include <v8.h>
#include <sstream>
#include <string.h>
#include "rapidxml.hpp"

// Used example code from:
// https://github.com/glynos/cpp-netlib/blob/master/contrib/http_examples/rss/rss.cpp
// http://nodejs.org/api/addons.html

using namespace v8;
using namespace rapidxml;

char const *EMPTY_C_STRING = "";

// Helper to read text node value.
// Returns 0 when cannot read the value.

char const *readTextNode(xml_node<char> *rootNode, const char* name) {
    xml_node<char> *node = rootNode->first_node(name);
    if (node) {
        xml_node<char> *textNode = node->first_node();
        if (textNode) {
            return textNode->value();
        } else {
            return EMPTY_C_STRING;
        }
    } else {
        return 0;
    }
}

// Helper to find the line number of error.

std::pair<int, int> findErrorLine(const char* xml, const char* where) {
    int i = 0;
    int ln = 1;
    char ch = 0;
    int col = 1;
    while (true) {
        if (xml + i == where) {
            break;
        }
        ch = xml[i];
        if (ch == '\n') {
            ln++;
            col = 0;
        } else {
            col++;
        }
        i++;
    }
    return std::pair<int, int>(ln, col);
}

// Parses the Atom feed.

void parseAtomFeed(xml_node<char> *feedNode, const Local<Object> &feed, bool extractContent) {

    feed->Set(String::NewSymbol("type"), String::New("atom"));

    // Extracts the title property.

    char const *title = readTextNode(feedNode, "title");
    if (title) {
        feed->Set(String::NewSymbol("title"), String::New(title));
    }

    // Extracts the id property.

    char const *id = readTextNode(feedNode, "id");
    if (id) {
        feed->Set(String::NewSymbol("id"), String::New(id));
    }

    // Extracts the link property.
    xml_node<char> *linkNode = feedNode->first_node("link");
    if (linkNode) {
        xml_attribute<char> *hrefAttr = linkNode->first_attribute("href");
        if (hrefAttr) {
            feed->Set(String::NewSymbol("link"), String::New(hrefAttr->value()));
        }
    }

    // Extracts the author property.

    char const *author = readTextNode(feedNode, "author");
    if (author) {
        feed->Set(String::NewSymbol("author"), String::New(author));
    }

    // Extract all channel items.

    Local<Array> items = Array::New();
    xml_node<char> *itemNode = feedNode->first_node("entry");

    int i = 0;
    while (itemNode) {
        Local<Object> item = Object::New();

        // Extracts the id property.

        char const *id = readTextNode(itemNode, "id");
        if (id) {
            item->Set(String::NewSymbol("id"), String::New(id));
        }

        Local<Array> links = Array::New();

        // Extracts all links.
        // 4.2.7. The "atom:link" Element

        xml_node<char> *linkNode = itemNode->first_node("link");

        int linkIndex = 0;

        while (linkNode) {

            Local<Object> link = Object::New();

            xml_attribute<char> *relAttr = linkNode->first_attribute("rel");
            xml_attribute<char> *hrefAttr = linkNode->first_attribute("href");
            xml_attribute<char> *typeAttr = linkNode->first_attribute("type");
            xml_attribute<char> *hreflangAttr = linkNode->first_attribute("hreflang");
            xml_attribute<char> *titleAttr = linkNode->first_attribute("title");
            xml_attribute<char> *lengthAttr = linkNode->first_attribute("length");

            if (relAttr) {
                link->Set(String::NewSymbol("rel"), String::New(relAttr->value()));
            }

            if (hrefAttr) {
                link->Set(String::NewSymbol("href"), String::New(hrefAttr->value()));
            }

            if (typeAttr) {
                link->Set(String::NewSymbol("type"), String::New(typeAttr->value()));
            }

            if (hreflangAttr) {
                link->Set(String::NewSymbol("hreflang"), String::New(hreflangAttr->value()));
            }

            if (titleAttr) {
                link->Set(String::NewSymbol("title"), String::New(titleAttr->value()));
            }

            if (lengthAttr) {
                link->Set(String::NewSymbol("length"), String::New(lengthAttr->value()));
            }

            xml_node<char> *textNode = linkNode->first_node();

            // This is not by spec but some feeds
            // put URL/IRI into link's text node like:
            // <link>http://example.com</link>

            if (textNode) {
                link->Set(String::NewSymbol("text"), String::New(textNode->value()));
            }

            links->Set(Number::New(linkIndex), link);

            linkIndex++;

            linkNode = linkNode->next_sibling("link");
        }

        item->Set(String::NewSymbol("links"), links);

        // Extract the item title.

        char const *title = readTextNode(itemNode, "title");
        if (title) {
            item->Set(String::NewSymbol("title"), String::New(title));
        }

        // Extract the published property.

{'fraction_non_alphanumeric': 0.15746298519407764, 'fraction_numerical': 0.0028011204481792717, 'mean_word_length': 2.232137083737472, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 135, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 40}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '451978', 'n_tokens_mistral': 3082, 'n_tokens_neox': 2950, 'n_words': 639}