<commit_before>var fs = require('fs'),
    util = require('util'),
    jison = require('jison');

var lexer = require('./lib/parser/bison/lexer'),
    grammar = require('./lib/parser/bison/grammar'),
    operators = require('./lib/parser/bison/operators');

var gen = new jison.Generator({
    lex: lexer,
    bnf: grammar,
    operators: operators.reverse(),
    startSymbol: 'Root'
});

var source = gen.generateModule();
source += '\nvar nodes = require("./nodes/all");\nexports.parse = function(source) { var p = new parser.Parser(); p.yy = nodes; return p.parse(source);};';
fs.writeFileSync('lib/parser/parser.generated.js', source);


// Test -----------------------------------------------------------------------
// ----------------------------------------------------------------------------

// Some simple tests
var Ast = require('./lib/parser/Ast');

var source = 'l += (!foo.bar[2]((2 ** 2 // 2), foo, 4))[2:@foo]';
var tree = new Ast(source);
console.log(util.inspect(tree.tree, false, 10));

//console.log(tree.toString());

tree.visit(function(node, parent, depth) {
    console.log(new Array(depth * 4).join(' '), ' -', node.type, '=>', node.toString());

}, true);

//var code = '~!(+true) + (-2) * (++e) / 5 * foo()[2..(1 - 3)] + @foo.bar[2 + 2] - (a ? b : c) // 2 + @ + a > b && c < d || 4 != 5 - foo(1, 2, 3, foo = 2 + 2, bla...)';
<commit_msg>[UPDATE] Make use of the compiler for testing<commit_after>var fs = require('fs'),
    util = require('util'),
    jison = require('jison');

var lexer = require('./lib/parser/bison/lexer'),
    grammar = require('./lib/parser/bison/grammar'),
    operators = require('./lib/parser/bison/operators');

var gen = new jison.Generator({
    lex: lexer,
    bnf: grammar,
    operators: operators.reverse(),
    startSymbol: 'Root'
});

var source = gen.generateModule();
source += '\nvar nodes = require("./nodes/all");\nexports.parse = function(source) { var p = new parser.Parser(); p.yy = nodes; return p.parse(source);};';
fs.writeFileSync('lib/parser/parser.generated.js', source);


// Test -----------------------------------------------------------------------
// ----------------------------------------------------------------------------

// Some simple tests
function testAST() {

    var Ast = require('./lib/parser/Ast');

    var source = 'l += (!foo.bar[2]((2 ** 2 // 2), foo, 4))[2:@foo]';
    var tree = new Ast(source);
    console.log(util.inspect(tree.tree, false, 10));

    //console.log(tree.toString());

    tree.visit(function(node, parent, depth) {
        console.log(new Array(depth * 4).join(' '), ' -', node.type, '=>', node.toString());

    }, true);

    //var code = '~!(+true) + (-2) * (++e) / 5 * foo()[2..(1 - 3)] + @foo.bar[2 + 2] - (a ? b : c) // 2 + @ + a > b && c < d || 4 != 5 - foo(1, 2, 3, foo = 2 + 2, bla...)';


}

var Compiler = require('./lib/compiler/Compiler');

var emblem = new Compiler();

// Test name resolving
emblem.compile('modules.mod'); // File
emblem.compile('modules.bar.test'); // File
emblem.compile('modules.bar.ignore'); // file
emblem.compile('modules.bar.lum'); // Index
emblem.compile('modules.bar.foo.test'); // File

{'fraction_non_alphanumeric': 0.30256898192197906, 'fraction_numerical': 0.015223596574690771, 'mean_word_length': 2.2683937823834195, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1899702', 'n_tokens_mistral': 1055, 'n_tokens_neox': 1005, 'n_words': 235}