<commit_before>//node --prof --prof_auto profile.js
//deps/v8/tools/mac-tick-processor v8.log
var sys = require("sys");
var fs = require("fs");
var http = require("http");
var htmlparser = require("./node-htmlparser");
var libxml = require("./libxmljs");

function GetTime() {
    return new Date().getTime();
}

function TimeCode(loops, func) {
    var start = GetTime();

    while (loops--) func();

    return GetTime() - start;
}

var html = fs.readFileSync("testdata/trackerchecker.html");

var timeNodeHtmlParser = TimeCode(100, function() {
    var handler = new htmlparser.DefaultHandler(function(err, dom) {
        if (err) sys.debug("Error: " + err);
        //	else
        //		sys.debug(sys.inspect(dom, false, null));
    });
    var parser = new htmlparser.Parser(handler);
    parser.ParseComplete(html);
});

var timeLibXmlJs = TimeCode(100, function() {
    var dom = libxml.parseHtmlString(html);
});

sys.debug("NodeHtmlParser: " + timeNodeHtmlParser);
sys.debug("LibXmlJs: " + timeLibXmlJs);
sys.debug(
    "Difference: " + ((timeNodeHtmlParser - timeLibXmlJs) / timeLibXmlJs) * 100
);<commit_msg>Updated profile.js to load a live html page<commit_after>//node --prof --prof_auto profile.js
//deps/v8/tools/mac-tick-processor v8.log
var sys = require("sys");
var fs = require("fs");
var http = require("http");
var htmlparser = require("./node-htmlparser");
var libxml = require("./libxmljs");

function GetTime() {
    return new Date().getTime();
}

function TimeCode(loops, func) {
    var start = GetTime();

    while (loops--) func();

    return GetTime() - start;
}

var html = "";
http.createClient(80, "en.wikipedia.org")
    .request("GET", "/wiki/Javascript", { host: "en.wikipedia.org" })
    .addListener("response", function(response) {
        if (response.statusCode == "200") {
            response.setEncoding("utf8");
            response
                .addListener("data", function(chunk) {
                    html += chunk;
                })
                .addListener("end", function() {
                    var timeNodeHtmlParser = TimeCode(100, function() {
                        var handler = new htmlparser.DefaultHandler(function(
                            err,
                            dom
                        ) {
                            if (err) sys.debug("Error: " + err);
                        });
                        var parser = new htmlparser.Parser(handler);
                        parser.ParseComplete(html);
                    });

                    var timeLibXmlJs = TimeCode(100, function() {
                        var dom = libxml.parseHtmlString(html);
                    });

                    sys.debug("NodeHtmlParser: " + timeNodeHtmlParser);
                    sys.debug("LibXmlJs: " + timeLibXmlJs);
                    sys.debug(
                        "Difference: " +
                            ((timeNodeHtmlParser - timeLibXmlJs) /
                                timeLibXmlJs) *
                                100
                    );
                });
        } else sys.debug("Error: got response status " + response.statusCode);
    })
    .end();
{'fraction_non_alphanumeric': 0.14096499526963102, 'fraction_numerical': 0.008830022075055188, 'mean_word_length': 1.8576576576576576, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2695724', 'n_tokens_mistral': 915, 'n_tokens_neox': 872, 'n_words': 189}