<commit_before>/*
Copyright (c) 2015, Yahoo! Inc. All rights reserved.
Copyrights licensed under the New BSD License.
See the accompanying LICENSE file for terms.

Authors: <NAME> <<EMAIL>>
         <NAME> <<EMAIL>>
         <NAME> <<EMAIL>>
*/
/*
This utility print out the state and related information of each character of the HTML5 web page.
*/
var Debug = require("debug"),
    progname = 'HTML-State';

Debug.enable(progname);

(function() {
    var Parser = require("./context-parser").Parser,
        debug = Debug(progname),
        fs = require('fs'),
        file,
        noofargs = 0,
        parser = new Parser();

    process.argv.forEach(function(val, index) {
        ++noofargs;
        if (index === 2) {
            file = val;
        }
    });

    Parser.prototype.printCharWithState = function() {
        var len = this.states.length;
        debug('{ statesSize: '+len+' }');
        for(var i=0;i<len;i++) {
            var c = this.bytes[i];
            var s = this.states[i];
            var t = this.symbols[i];

            if (i === 0) {
                debug('{ ch: 0, state: '+s+', symbol: 0 }');
            } else if ( t === 0 ) {
                /* space or eol */
                debug('{ ch:   [0x'+c.charCodeAt(0).toString(16)+'], state: '+s+', symbol: '+t+' }');
            } else {
                if (c !== undefined) {
                    debug('{ ch: '+c+' [0x'+c.charCodeAt(0).toString(16)+'], state: '+s+', symbol: '+t+' }');
                } else {
                    debug('{ undefined - char in template without state }');
                }
            }
        }
        return 0;
    };

    if (noofargs === 3) {
        if (fs.existsSync(file)) {
            parser.printCharWithState();
            process.exit(0);
        } else {
            console.log("[ERROR] "+file+" not exist");
            process.exit(1);
        }
    } else {
        console.log("Usage: contextparse <any template file>");
        process.exit(1);
    }

}).call(this);
<commit_msg>add contextualize back to the parser<commit_after>/*
Copyright (c) 2015, Yahoo! Inc. All rights reserved.
Copyrights licensed under the New BSD License.
See the accompanying LICENSE file for terms.

Authors: <NAME> <<EMAIL>>
         <NAME> <<EMAIL>>
         <NAME> <<EMAIL>>
*/
/*
This utility print out the state and related information of each character of the HTML5 web page.
*/
var Debug = require("debug"),
    progname = 'HTML-State';

Debug.enable(progname);

(function() {
    var Parser = require("./context-parser").Parser,
        debug = Debug(progname),
        fs = require('fs'),
        file,
        noofargs = 0,
        parser = new Parser();

    process.argv.forEach(function(val, index) {
        ++noofargs;
        if (index === 2) {
            file = val;
        }
    });

    Parser.prototype.printCharWithState = function() {
        var len = this.states.length;
        debug('{ statesSize: '+len+' }');
        for(var i=0;i<len;i++) {
            var c = this.bytes[i];
            var s = this.states[i];
            var t = this.symbols[i];

            if (i === 0) {
                debug('{ ch: 0, state: '+s+', symbol: 0 }');
            } else if ( t === 0 ) {
                /* space or eol */
                debug('{ ch:   [0x'+c.charCodeAt(0).toString(16)+'], state: '+s+', symbol: '+t+' }');
            } else {
                if (c !== undefined) {
                    debug('{ ch: '+c+' [0x'+c.charCodeAt(0).toString(16)+'], state: '+s+', symbol: '+t+' }');
                } else {
                    debug('{ undefined - char in template without state }');
                }
            }
        }
        return 0;
    };

    if (noofargs === 3) {
        if (fs.existsSync(file)) {
            var data = fs.readFileSync(file, 'utf-8');
            parser.contextualize(data, data.length);            
            parser.printCharWithState();
            process.exit(0);
        } else {
            console.log("[ERROR] "+file+" not exist");
            process.exit(1);
        }
    } else {
        console.log("Usage: contextparse <any template file>");
        process.exit(1);
    }

}).call(this);

{'fraction_non_alphanumeric': 0.16535433070866143, 'fraction_numerical': 0.012168933428775949, 'mean_word_length': 1.7132686084142394, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6301821', 'n_tokens_mistral': 1319, 'n_tokens_neox': 1249, 'n_words': 333}