<commit_before>
var cobs = require('../'),
    assert = require('assert');
    
var TokenType = cobs.TokenType;
    
// Lexer defined

assert.ok(cobs.Lexer);

// Null token if null text

var lexer = new cobs.Lexer(null);

assert.equal(null, lexer.nextToken());

// Null token if empty text

var lexer = new cobs.Lexer('');

assert.equal(null, lexer.nextToken());

// Get simple name

var lexer = new cobs.Lexer('DIVISION');

var token = lexer.nextToken();
assert.ok(token);
assert.equal('DIVISION', token.value);
assert.equal(TokenType.Name, token.type);

assert.equal(null, lexer.nextToken());

// Get simple name with digits

var lexer = new cobs.Lexer('ITEM01');

var token = lexer.nextToken();
assert.ok(token);
assert.equal('ITEM01', token.value);
assert.equal(TokenType.Name, token.type);

assert.equal(null, lexer.nextToken());

// Get two simple names

var lexer = new cobs.Lexer('IDENTIFICATION DIVISION');

var token = lexer.nextToken();
assert.ok(token);
assert.equal('IDENTIFICATION', token.value);
assert.equal(TokenType.Name, token.type);

token = lexer.nextToken();
assert.ok(token);
assert.equal('DIVISION', token.value);
assert.equal(TokenType.Name, token.type);

assert.equal(null, lexer.nextToken());

// Get name with minus sign

var lexer = new cobs.Lexer('WORKING-STORAGE');

var token = lexer.nextToken();
assert.ok(token);
assert.equal('WORKING-STORAGE', token.value);
assert.equal(TokenType.Name, token.type);

assert.equal(null, lexer.nextToken());<commit_msg>Test refactor<commit_after>
var cobs = require('../'),
    assert = require('assert');
    
var TokenType = cobs.TokenType;

function assertToken(lexer, value, type)
{
    var token = lexer.nextToken();
    assert.ok(token);
    assert.equal(value, token.value);
    assert.equal(type, token.type);
}
    
// Lexer defined

assert.ok(cobs.Lexer);

// Null token if null text

var lexer = new cobs.Lexer(null);

assert.equal(null, lexer.nextToken());

// Null token if empty text

var lexer = new cobs.Lexer('');

assert.equal(null, lexer.nextToken());

// Get simple name

var lexer = new cobs.Lexer('DIVISION');

assertToken(lexer, 'DIVISION', TokenType.Name);

assert.equal(null, lexer.nextToken());

// Get simple name with digits

var lexer = new cobs.Lexer('ITEM01');

assertToken(lexer, 'ITEM01', TokenType.Name);

assert.equal(null, lexer.nextToken());

// Get two simple names

var lexer = new cobs.Lexer('IDENTIFICATION DIVISION');

assertToken(lexer, 'IDENTIFICATION', TokenType.Name);
assertToken(lexer, 'DIVISION', TokenType.Name);

assert.equal(null, lexer.nextToken());

// Get name with minus sign

var lexer = new cobs.Lexer('WORKING-STORAGE');

assertToken(lexer, 'WORKING-STORAGE', TokenType.Name);

assert.equal(null, lexer.nextToken());
{'fraction_non_alphanumeric': 0.16301969365426697, 'fraction_numerical': 0.0029175784099197666, 'mean_word_length': 4.762605042016807, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5776860', 'n_tokens_mistral': 1010, 'n_tokens_neox': 947, 'n_words': 218}