<commit_before>var index = require("./index.js"),
    DomHandler = index.DomHandler,
    DomUtils = index.DomUtils;

//TODO: make this a streamable handler
function FeedHandler(callback) {
    this.init(callback, { ignoreWhitespace: true });
}

require("util").inherits(FeedHandler, DomHandler);

FeedHandler.prototype.init = DomHandler;

function getElements(what, where, one, recurse) {
    if (one) return DomUtils.getElementsByTagName(what, where, recurse, 1)[0];
    return DomUtils.getElementsByTagName(what, where, recurse);
}
function fetch(what, where, recurse) {
    var ret = DomUtils.getElementsByTagName(what, where, recurse, 1);
    return (
        ret.length > 0 && ret[0].children.length > 0 && ret[0].children[0].data
    );
}

var isValidFeed = function(value) {
    return value === "rss" || value === "feed" || value === "rdf:RDF";
};

FeedHandler.prototype.onend = function() {
    var feed = {},
        feedRoot = getElements(isValidFeed, this.dom, true),
        tmp,
        childs;

    if (feedRoot) {
        if (feedRoot.name === "feed") {
            childs = feedRoot.children;

            feed.type = "atom";
            if ((tmp = fetch("id", childs))) feed.id = tmp;
            if ((tmp = fetch("title", childs))) feed.title = tmp;
            if (
                (tmp = getElements("link", childs, true)) &&
                (tmp = tmp.attribs) &&
                (tmp = tmp.href)
            )
                feed.link = tmp;
            if ((tmp = fetch("subtitle", childs))) feed.description = tmp;
            if ((tmp = fetch("updated", childs))) feed.updated = new Date(tmp);
            if ((tmp = fetch("email", childs, true))) feed.author = tmp;

            feed.items = getElements("entry", childs).map(function(item) {
                var entry = {},
                    tmp;

                item = item.children;

                if ((tmp = fetch("id", item))) entry.id = tmp;
                if ((tmp = fetch("title", item))) entry.title = tmp;
                if (
                    (tmp = getElements("link", item, true)) &&
                    (tmp = tmp.attribs) &&
                    (tmp = tmp.href)
                )
                    entry.link = tmp;
                if ((tmp = fetch("summary", item))) entry.description = tmp;
                if ((tmp = fetch("updated", item)))
                    entry.pubDate = new Date(tmp);
                return entry;
            });
        } else {
            childs = getElements("channel", feedRoot.children, true).children;

            feed.type = feedRoot.name.substr(0, 3);
            feed.id = "";
            if ((tmp = fetch("title", childs))) feed.title = tmp;
            if ((tmp = fetch("link", childs))) feed.link = tmp;
            if ((tmp = fetch("description", childs))) feed.description = tmp;
            if ((tmp = fetch("lastBuildDate", childs)))
                feed.updated = new Date(tmp);
            if ((tmp = fetch("managingEditor", childs))) feed.author = tmp;

            feed.items = getElements("item", feedRoot.children).map(function(
                item
            ) {
                var entry = {},
                    tmp;

                item = item.children;

                if ((tmp = fetch("guid", item))) entry.id = tmp;
                if ((tmp = fetch("title", item))) entry.title = tmp;
                if ((tmp = fetch("link", item))) entry.link = tmp;
                if ((tmp = fetch("description", item))) entry.description = tmp;
                if ((tmp = fetch("pubDate", item)))
                    entry.pubDate = new Date(tmp);
                return entry;
            });<commit_msg>use DomUtils.getText in fetch, split getElements<commit_after>var index = require("./index.js"),
    DomHandler = index.DomHandler,
    DomUtils = index.DomUtils;

//TODO: make this a streamable handler
function FeedHandler(callback) {
    this.init(callback, { ignoreWhitespace: true });
}

require("util").inherits(FeedHandler, DomHandler);

FeedHandler.prototype.init = DomHandler;

function getElements(what, where) {
    return DomUtils.getElementsByTagName(what, where, false);
}
function getOneElement(what, where) {
    return DomUtils.getElementsByTagName(what, where, false, 1)[0];
}
function fetch(what, where, recurse) {
    return DomUtils.getText(
        DomUtils.getElementsByTagName(what, where, !!recurse, 1)
    );
}

var isValidFeed = function(value) {
    return value === "rss" || value === "feed" || value === "rdf:RDF";
};

FeedHandler.prototype.onend = function() {
    var feed = {},
        feedRoot = getOneElement(isValidFeed, this.dom),
        tmp,
        childs;

    if (feedRoot) {
        if (feedRoot.name === "feed") {
            childs = feedRoot.children;

            feed.type = "atom";
            if ((tmp = fetch("id", childs))) feed.id = tmp;
            if ((tmp = fetch("title", childs))) feed.title = tmp;
            if (
                (tmp = getOneElement("link", childs)) &&
                (tmp = tmp.attribs) &&
                (tmp = tmp.href)
            )
                feed.link = tmp;
            if ((tmp = fetch("subtitle", childs))) feed.description = tmp;
            if ((tmp = fetch("updated", childs))) feed.updated = new Date(tmp);
            if ((tmp = fetch("email", childs, true))) feed.author = tmp;

            feed.items = getElements("entry", childs).map(function(item) {
                var entry = {},
                    tmp;

                item = item.children;

                if ((tmp = fetch("id", item))) entry.id = tmp;
                if ((tmp = fetch("title", item))) entry.title = tmp;
                if (
                    (tmp = getOneElement("link", item)) &&
                    (tmp = tmp.attribs) &&
                    (tmp = tmp.href)
                )
                    entry.link = tmp;
                if ((tmp = fetch("summary", item))) entry.description = tmp;
                if ((tmp = fetch("updated", item)))
                    entry.pubDate = new Date(tmp);
                return entry;
            });
        } else {
            childs = getOneElement("channel", feedRoot.children).children;

            feed.type = feedRoot.name.substr(0, 3);
            feed.id = "";
            if ((tmp = fetch("title", childs))) feed.title = tmp;
            if ((tmp = fetch("link", childs))) feed.link = tmp;
            if ((tmp = fetch("description", childs))) feed.description = tmp;
            if ((tmp = fetch("lastBuildDate", childs)))
                feed.updated = new Date(tmp);
            if ((tmp = fetch("managingEditor", childs))) feed.author = tmp;

            feed.items = getElements("item", feedRoot.children).map(function(
                item
            ) {
                var entry = {},
                    tmp;

                item = item.children;

                if ((tmp = fetch("guid", item))) entry.id = tmp;
                if ((tmp = fetch("title", item))) entry.title = tmp;
                if ((tmp = fetch("link", item))) entry.link = tmp;
                if ((tmp = fetch("description", item))) entry.description = tmp;
                if ((tmp = fetch("pubDate", item)))
                    entry.pubDate = new Date(tmp);
                return entry;
            });
{'fraction_non_alphanumeric': 0.14921070693205216, 'fraction_numerical': 0.002059025394646534, 'mean_word_length': 1.7661351556567957, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1502713', 'n_tokens_mistral': 2059, 'n_tokens_neox': 1970, 'n_words': 502}