<commit_before>/*jslint node: true */
"use strict";

var vows = require("vows"),
    assert = require("assert");

function parse(file, expected) {
    var lexer = require("../source/lexer"),
        parser = require("../source/parser"),
        fs = require("fs"),
        path = require("path");

    return {

        topic: function () {
            var callback = this.callback;
            fs.readFile(path.resolve("./test/code", file), "utf8",
                function (err, code) {
                    callback(err, parser.parse(lexer.tokenize(code)));
                });
        },

        "Generates correct ast": function (err, nodes) {
            assert.ifError(err);
            assert.deepEqual(nodes, expected);
        }

    };
}

vows.describe("Parser").addBatch({

    "Hello world": parse("hello-world.code", [{"type": "CALL",
        "identifier": "print", "arguments": [{"type": "TEXT",
        "value": "hello world"}]}]),

    "If": parse(
        "if.code",
       [{
          "type":"SET_LOCAL_VALUE",
          "identifier":"variable",
          "value":{
             "type":"TEXT",
             "value":"lemon"
          }
       },
       {
          "type":"IF",
          "condition":{
             "type":"CALL",
             "identifier":"is",
             "arguments":[
                {
                   "type":"GET_LOCAL_VALUE",
                   "identifier":"variable"
                },
                {
                   "type":"TEXT",
                   "value":"lemon"
                }
             ]
          },
          "body":[
             {
                "type":"CALL",
                "identifier":"print",
                "arguments":[
                   {
                      "type":"TEXT",
                      "value":"true!"
                   }
                ]
             }
          ]
       }])

}).exportTo(module);

<commit_msg>generate parser tests automatically<commit_after>/*jslint node: true */
"use strict";

var vows = require("vows"),
    assert = require("assert"),
    fs = require("fs"),
    path = require("path"),
    lexer = require("../source/lexer"),
    parser = require("../source/parser"),
    tests = {};

/**
 * Parse the code and compare the resulting AST with the expected one.
 */
function parse(name) {

    var code = path.resolve("./test/code", name + ".code"),
        expected = path.resolve("./test/parser", name + ".json");

    return {

        topic: function () {
            var callback = this.callback;
            fs.readFile(code, "utf8",
                function (err, code) {
                    callback(err, parser.parse(lexer.tokenize(code)));
                });
        },

        "Generates correct ast": function (err, nodes) {
            assert.ifError(err);
            assert.deepEqual(nodes, require(expected));
        }

    };

}

// Generate test for each code file
fs.readdirSync("./test/code").forEach(function (file) {
    var name = path.basename(file, ".code");
    tests[name] = parse(name);
});

vows.describe("Parser").addBatch(tests).exportTo(module);


{'fraction_non_alphanumeric': 0.1667741935483871, 'fraction_numerical': 0.0006451612903225806, 'mean_word_length': 1.5376432078559739, 'pattern_counts': {'":': 29, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7609526', 'n_tokens_mistral': 872, 'n_tokens_neox': 827, 'n_words': 165}