<commit_before>/*jslint node: true */
"use strict";

var vows = require("vows"),
    assert = require("assert"),
    lexer = require("../source/lexer"),
    fs = require("fs"),
    path = require("path"),
    tests = {};

function lex(name) {

    var code = path.resolve("./test/code", name + ".code"),
        expected = path.resolve("./test/lexer", name + ".json");

    return {

        topic: function () {
            var callback = this.callback;
            fs.readFile(code, "utf8",
                function (err, code) {
                    callback(err, lexer.tokenize(code));<commit_msg>added comments<commit_after>/*jslint node: true */
"use strict";

var vows = require("vows"),
    assert = require("assert"),
    lexer = require("../source/lexer"),
    fs = require("fs"),
    path = require("path"),
    tests = {};

/**
 * Tokenize the code and compare the results to the expected.
 */
function lex(name) {

    var code = path.resolve("./test/code", name + ".code"),
        expected = path.resolve("./test/lexer", name + ".json");

    return {

        topic: function () {
            var callback = this.callback;
            fs.readFile(code, "utf8",
                function (err, code) {
                    callback(err, lexer.tokenize(code));
{'fraction_non_alphanumeric': 0.17387883556254918, 'fraction_numerical': 0.0015735641227380016, 'mean_word_length': 2.2448979591836733, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5953093', 'n_tokens_mistral': 388, 'n_tokens_neox': 371, 'n_words': 92}