<commit_before>var test = module.exports = {},
    fs = require('fs'),
    assert = require('assert'),
    helper = require(__dirname + '/../test-helper'),
    jsyaml = require(__dirname + '/../../lib/js-yaml'),
    $$ = require(__dirname + '/../../lib/js-yaml/core'),
    __ = $$.import('nodes');

// Tokens mnemonic:
// directive:            %
// document_start:       ---
// document_end:         ...
// alias:                *
// anchor:               &
// tag:                  !
// scalar                _
// block_sequence_start: [[
// block_mapping_start:  {{
// block_end:            ]}
// flow_sequence_start:  [
// flow_sequence_end:    ]
// flow_mapping_start:   {
// flow_mapping_end:     }
// entry:                ,
// key:                  ?
// value:                :

var REPLACES = new $$.Hash();

REPLACES.store(__.DirectiveToken,           '%');
REPLACES.store(__.DocumentStartToken,       '---');<commit_msg>fixed import of tokens test<commit_after>var test = module.exports = {},
    fs = require('fs'),
    assert = require('assert'),
    helper = require(__dirname + '/../test-helper'),
    jsyaml = require(__dirname + '/../../lib/js-yaml'),
    $$ = require(__dirname + '/../../lib/js-yaml/core'),
    __ = $$.import('tokens');

// Tokens mnemonic:
// directive:            %
// document_start:       ---
// document_end:         ...
// alias:                *
// anchor:               &
// tag:                  !
// scalar                _
// block_sequence_start: [[
// block_mapping_start:  {{
// block_end:            ]}
// flow_sequence_start:  [
// flow_sequence_end:    ]
// flow_mapping_start:   {
// flow_mapping_end:     }
// entry:                ,
// key:                  ?
// value:                :

var REPLACES = new $$.Hash();

REPLACES.store(__.DirectiveToken,           '%');
REPLACES.store(__.DocumentStartToken,       '---');
{'fraction_non_alphanumeric': 0.192, 'fraction_numerical': 0.0, 'mean_word_length': 1.5454545454545454, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5192496', 'n_tokens_mistral': 579, 'n_tokens_neox': 560, 'n_words': 86}