<commit_before>var test = module.exports = {},
    fs = require('fs'),
    assert = require('assert'),
    helper = require(__dirname + '/../test-helper'),
    jsyaml = require(__dirname + '/../../lib/js-yaml'),
    $$ = require(__dirname + '/../../lib/js-yaml/core'),
    __ = $$.import('tokens');

// Tokens mnemonic:
// directive:            %
// document_start:       ---
// document_end:         ...
// alias:                *
// anchor:               &
// tag:                  !
// scalar                _
// block_sequence_start: [[
// block_mapping_start:  {{
// block_end:            ]}
// flow_sequence_start:  [
// flow_sequence_end:    ]
// flow_mapping_start:   {
// flow_mapping_end:     }
// entry:                ,
// key:                  ?
// value:                :

var REPLACES = new $$.Hash();

REPLACES.store(__.DirectiveToken,           '%');
REPLACES.store(__.DocumentStartToken,       '---');
REPLACES.store(__.DocumentEndToken,         '...');
REPLACES.store(__.AliasToken,               '*');
REPLACES.store(__.AnchorToken,              '&');
REPLACES.store(__.TagToken,                 '!');
REPLACES.store(__.ScalarToken,              '_');
REPLACES.store(__.BlockSequenceStartToken,  '[[');
REPLACES.store(__.BlockMappingStartToken,   '{{');
REPLACES.store(__.BlockEndToken,            ']}');
REPLACES.store(__.FlowSequenceStartToken,   '[');
REPLACES.store(__.FlowSequenceEndToken,     ']');
REPLACES.store(__.FlowMappingStartToken,    '{');
REPLACES.store(__.FlowMappingEndToken,      '}');
REPLACES.store(__.BlockEntryToken,          ',');
REPLACES.store(__.FlowEntryToken,           ',');
REPLACES.store(__.KeyToken,                 '?');
REPLACES.store(__.ValueToken,               ':');


var test_tokens = function test_tokens(data_filename, tokens_filename) {
  var tokens1 = [],
      tokens2 = [];
  
  fs.readFileSync(tokens_filename, 'utf8').split(/[ \n]/).forEach(function (t) {
    if (!!t) { tokens2.push(t); }
  });

  jsyaml.scan(fs.readFileSync(data_filename, 'utf8'), function (token) {
    if (token.isA(__.StreamStartToken) || token.isA(__.StreamEndToken)) {
      return;
    }

    tokens1.push(REPLACES.get(token.__klass__));
  });

  assert.equal(tokens1.length, tokens2.length);

  tokens1.forEach(function (token1, i) {
    assert.equal(token1, tokens2[i]);
  });
};


test.title = "Test tokens generation";
test.execute = function () {
  $$.each(helper.findTestFilenames(__dirname + '/data'), function (exts, base) {
    var files = [];

    $$.each(['.data', '.tokens'], function (ext) {
      if (0 <= exts.indexOf(ext)) {
        files.push(__dirname + '/data/' + base + ext);
      }
    });

    if (2 === files.length) {
      test_tokens.apply(this, files);
    }
  });
};

////////////////////////////////////////////////////////////////////////////////
// vim:ts=2:sw=2
////////////////////////////////////////////////////////////////////////////////
<commit_msg>Fixed tokens test<commit_after>var test = module.exports = {},
    fs = require('fs'),
    assert = require('assert'),
    jsyaml = require(__dirname + '/../../lib/js-yaml'),
    helper = require(__dirname + '/../test-helper'),
    $$ = require(__dirname + '/../../lib/js-yaml/core'),
    __ = $$.import('tokens');

// Tokens mnemonic:
// directive:            %
// document_start:       ---
// document_end:         ...
// alias:                *
// anchor:               &
// tag:                  !
// scalar                _
// block_sequence_start: [[
// block_mapping_start:  {{
// block_end:            ]}
// flow_sequence_start:  [
// flow_sequence_end:    ]
// flow_mapping_start:   {
// flow_mapping_end:     }
// entry:                ,
// key:                  ?
// value:                :

var REPLACES = new $$.Hash();

REPLACES.store(__.DirectiveToken,           '%');
REPLACES.store(__.DocumentStartToken,       '---');
REPLACES.store(__.DocumentEndToken,         '...');
REPLACES.store(__.AliasToken,               '*');
REPLACES.store(__.AnchorToken,              '&');
REPLACES.store(__.TagToken,                 '!');
REPLACES.store(__.ScalarToken,              '_');
REPLACES.store(__.BlockSequenceStartToken,  '[[');
REPLACES.store(__.BlockMappingStartToken,   '{{');
REPLACES.store(__.BlockEndToken,            ']}');
REPLACES.store(__.FlowSequenceStartToken,   '[');
REPLACES.store(__.FlowSequenceEndToken,     ']');
REPLACES.store(__.FlowMappingStartToken,    '{');
REPLACES.store(__.FlowMappingEndToken,      '}');
REPLACES.store(__.BlockEntryToken,          ',');
REPLACES.store(__.FlowEntryToken,           ',');
REPLACES.store(__.KeyToken,                 '?');
REPLACES.store(__.ValueToken,               ':');


test.unittest = ['.data', '.tokens'];
test.execute = function test_tokens(data_filename, tokens_filename) {
  var tokens1 = [],
      tokens2 = [];
  
  fs.readFileSync(tokens_filename, 'utf8').split(/[ \n]/).forEach(function (t) {
    if (!!t) { tokens2.push(t); }
  });

  jsyaml.scan(fs.readFileSync(data_filename, 'utf8'), function (token) {
    if (token.isA(__.StreamStartToken) || token.isA(__.StreamEndToken)) {
      return;
    }

    tokens1.push(REPLACES.get(token.klass));
  });

  assert.equal(tokens1.length, tokens2.length);

  tokens1.forEach(function (token1, i) {
    assert.equal(token1, tokens2[i]);
  });
};


////////////////////////////////////////////////////////////////////////////////
// vim:ts=2:sw=2
////////////////////////////////////////////////////////////////////////////////

{'fraction_non_alphanumeric': 0.23787823550856727, 'fraction_numerical': 0.005468465184104995, 'mean_word_length': 1.9389394750937332, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5692986', 'n_tokens_mistral': 1744, 'n_tokens_neox': 1707, 'n_words': 211}