<commit_before>// html utilities including a parser and a url generator.

var htmlparser=require("htmlparser2");
var request=require("./request");

var Hierarchy=
{
    "MAX_H": 9,

    "title": 0,
    "h1": 1,
    "h2": 2,
    "h3": 3,
    "h4": 4,
    "h5": 5,
    "h6": 6,
    "h7": 7,
    "h8": 8,
    "others": 9
};
function parse(content, callback, onerror)
{
    hStack=[];
    result=[];
    hrefList=[];
    for (var i=0; i<=Hierarchy.MAX_H; i++)
        result.push("");
    var parser = new htmlparser.Parser(
    {
    	onopentag: function(name, attribs)
        {
            var lcName=name.toLowerCase();
    		if (lcName in Hierarchy)
            {
                hStack.push(Hierarchy[lcName]);
            }
            else if (lcName=="a")
            {
                if (attribs.href!=null)
                {
                    // URL crawled!
                    hrefList.push(attribs.href);
                }
            }
    	},
    	ontext: function(text)
        {
            var h=hStack.length==0?Hierarchy.MAX_H:hStack[hStack.length-1];
            result[h]+=" "+text;
    	},
    	onclosetag: function(tagname)
        {
            var lcName=tagname.toLowerCase();
    		if (lcName in Hierarchy)
            {
                var t=hStack.pop();
                if (t!=Hierarchy[lcName])
                {
                    console.warn((new Date()).toUTCString(), ">\tOut of order when parsing: "+tagname);
                    if (t!=null)
                        hStack.push(t);
                }
            }
    	},
        onend: function()
        {
            callback(result, hrefList);
        },
        onerror: function(err)
        {
            onerror(err);
        }
    }, {decodeEntities: true});
    parser.write(content);
    parser.end();
};

exports.parse=parse;
exports.Hierarchy=Hierarchy;


(function()
{
    request.GET("http://www.levy.at/me?lang=en-us", function(isCon, data)
    {
        console.log("stparse");
        parse(data, function(r, h)
        {
            console.log(r);
            console.log(h);
        });
    });<commit_msg>for debug<commit_after>// html utilities including a parser and a url generator.

var htmlparser=require("htmlparser2");
var request=require("./request");

var Hierarchy=
{
    "MAX_H": 9,

    "title": 0,
    "h1": 1,
    "h2": 2,
    "h3": 3,
    "h4": 4,
    "h5": 5,
    "h6": 6,
    "h7": 7,
    "h8": 8,
    "others": 9
};
function parse(content, callback, onerror)
{
    hStack=[];
    result=[];
    hrefList=[];
    for (var i=0; i<=Hierarchy.MAX_H; i++)
        result.push("");
    var parser = new htmlparser.Parser(
    {
    	onopentag: function(name, attribs)
        {
            var lcName=name.toLowerCase();
    		if (lcName in Hierarchy)
            {
                hStack.push(Hierarchy[lcName]);
            }
            else if (lcName=="a")
            {
                if (attribs.href!=null)
                {
                    // URL crawled!
                    hrefList.push(attribs.href);
                }
            }
    	},
    	ontext: function(text)
        {
            var h=hStack.length==0?Hierarchy.MAX_H:hStack[hStack.length-1];
            result[h]+=" "+text;
    	},
    	onclosetag: function(tagname)
        {
            var lcName=tagname.toLowerCase();
    		if (lcName in Hierarchy)
            {
                var t=hStack.pop();
                if (t!=Hierarchy[lcName])
                {
                    console.warn((new Date()).toUTCString(), ">\tOut of order when parsing: "+tagname);
                    if (t!=null)
                        hStack.push(t);
                }
            }
    	},
        onend: function()
        {
            callback(result, hrefList);
        },
        onerror: function(err)
        {
            onerror(err);
        }
    }, {decodeEntities: true});
    parser.write(content);
    parser.end();
};

exports.parse=parse;
exports.Hierarchy=Hierarchy;


(function()
{
    request.GET("http://edition.cnn.com", function(isCon, data)
    {
        parse(data, function(r, h)
        {
            console.log(r);
            console.log(h);
        });
    });
{'fraction_non_alphanumeric': 0.14947468958930277, 'fraction_numerical': 0.010983763132760267, 'mean_word_length': 1.4497076023391813, 'pattern_counts': {'":': 22, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '620848', 'n_tokens_mistral': 1404, 'n_tokens_neox': 1363, 'n_words': 250}