<commit_before>'use strict';

var Parser, sentence, paragraph, section, article, book;

Parser = require('..');

/* Test data */

/* Source: http://www.gutenberg.org/files/10745/10745-h/10745-h.htm */

/* A sentence, 20 words. */
sentence = 'Where she had stood was clear, and she was gone since Sir Kay does not choose to assume my quarrel.';

/* A paragraph, 5 sentences, 100 words. */
paragraph = 'Thou art a churlish knight to so affront a lady ' +
    'he could not sit upon his horse any longer. ' +
    'For methinks something hath befallen my lord and that he ' +
    'then, after a while, he cried out in great voice. ' +
    'For that light in the sky lieth in the south ' +
    'then Queen Helen fell down in a swoon, and lay. ' +
    'Touch me not, for I am not mortal, but Fay ' +
    'so the Lady of the Lake vanished away, everything behind. ' +
    sentence;

/* A section, 10 paragraphs, 50 sentences, 1,000 words. */
section = paragraph + Array(10).join('\n\n' + paragraph);

/* An article, 100 paragraphs, 500 sentences, 10,000 words. */
article = section + Array(10).join('\n\n' + section);

/* A book, 1,000 paragraphs, 5,000 sentences, 100,000 words. */
book = article + Array(10).join('\n\n' + article);

/* Benchmarks */
suite('parser.tokenizeSentence(source);', function () {
    var parser = new Parser();

    set('mintime', 50);

    bench('A sentence (20 words)', function (next) {
        parser.tokenizeSentence(sentence);
        next();
    });
});


/* Benchmarks */
suite('parser.tokenizeParagraph(source);', function () {
    var parser = new Parser();

    set('mintime', 50);

    bench('A sentence (20 words)', function (next) {
        parser.tokenizeParagraph(sentence);
        next();
    });

    bench('A paragraph (5 sentences, 100 words)', function (next) {
        parser.tokenizeParagraph(paragraph);
        next();
    });
});

/* Benchmarks */
suite('parser.tokenizeRoot(source);', function () {
    var parser = Parser();

    set('mintime', 100);

    bench('A paragraph (5 sentences, 100 words)', function (next) {
        parser.tokenizeRoot(paragraph);
        next();<commit_msg>Added a forgotten `new` token<commit_after>'use strict';

var Parser, sentence, paragraph, section, article, book;

Parser = require('..');

/* Test data */

/* Source: http://www.gutenberg.org/files/10745/10745-h/10745-h.htm */

/* A sentence, 20 words. */
sentence = 'Where she had stood was clear, and she was gone since Sir Kay does not choose to assume my quarrel.';

/* A paragraph, 5 sentences, 100 words. */
paragraph = 'Thou art a churlish knight to so affront a lady ' +
    'he could not sit upon his horse any longer. ' +
    'For methinks something hath befallen my lord and that he ' +
    'then, after a while, he cried out in great voice. ' +
    'For that light in the sky lieth in the south ' +
    'then Queen Helen fell down in a swoon, and lay. ' +
    'Touch me not, for I am not mortal, but Fay ' +
    'so the Lady of the Lake vanished away, everything behind. ' +
    sentence;

/* A section, 10 paragraphs, 50 sentences, 1,000 words. */
section = paragraph + Array(10).join('\n\n' + paragraph);

/* An article, 100 paragraphs, 500 sentences, 10,000 words. */
article = section + Array(10).join('\n\n' + section);

/* A book, 1,000 paragraphs, 5,000 sentences, 100,000 words. */
book = article + Array(10).join('\n\n' + article);

/* Benchmarks */
suite('parser.tokenizeSentence(source);', function () {
    var parser = new Parser();

    set('mintime', 50);

    bench('A sentence (20 words)', function (next) {
        parser.tokenizeSentence(sentence);
        next();
    });
});


/* Benchmarks */
suite('parser.tokenizeParagraph(source);', function () {
    var parser = new Parser();

    set('mintime', 50);

    bench('A sentence (20 words)', function (next) {
        parser.tokenizeParagraph(sentence);
        next();
    });

    bench('A paragraph (5 sentences, 100 words)', function (next) {
        parser.tokenizeParagraph(paragraph);
        next();
    });
});

/* Benchmarks */
suite('parser.tokenizeRoot(source);', function () {
    var parser = new Parser();

    set('mintime', 100);

    bench('A paragraph (5 sentences, 100 words)', function (next) {
        parser.tokenizeRoot(paragraph);
        next();
{'fraction_non_alphanumeric': 0.14951094550535632, 'fraction_numerical': 0.0367955286446204, 'mean_word_length': 3.1298076923076925, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1023566', 'n_tokens_mistral': 1452, 'n_tokens_neox': 1289, 'n_words': 454}