<commit_before>var tokenizer2 = require('tokenizer2');
var escapeRegExp = require('escape-regexp');

var special_chars = [
  ["(", "open"],
  ["[", "open"],
  ["{", "open"],

  [")", "close"],
  ["]", "close"],
  ["}", "close"],

  ["#", "dispatch"],
  ["'", "dispatch-quote"],
  ["`", "dispatch-super-quote"],
  ["@", "dispatch-deref"],
  ["~", "dispatch-unquote"],
  ["^", "dispatch-meta"]
];

var separators = '\\s' + escapeRegExp(',;"' + special_chars.map(function(d){
  return d[0];
}).join(''));

var number_regex = (function(){
  var s = '';
  s += '[+-]?';//may start with a + or -
  s += '(' + [
    '([0-9][0-9,]*\\.?[0-9]*)',//starting with a number
    '(\\.[0-9]+)'//starting with a period
  ].join('|') + ')';
  s += '([eE][-+]?[0-9,]*)?';//optional exponent
  return new RegExp('^' + s + '$');
}());

module.exports = function(){
  var t = tokenizer2();

  t.addRule(/^[\s,]+$/, 'whitespace');
  t.addRule(/^;[^\n]*$/, 'comment');
  t.addRule(/(^""$)|(^"([^"]|\\")*[^\\]"$)/, 'string');
  t.addRule(number_regex, 'number');

  special_chars.forEach(function(p){
    t.addRule(new RegExp('^' + escapeRegExp(p[0]) + '$'), p[1]);
  });

  t.addRule(new RegExp('^[^0-9' + separators + '][^' + separators + ']*$'), 'symbol');

  return t;
};<commit_msg>cleanup number token regexp<commit_after>var tokenizer2 = require('tokenizer2');
var escapeRegExp = require('escape-regexp');

var special_chars = [
  ["(", "open"],
  ["[", "open"],
  ["{", "open"],

  [")", "close"],
  ["]", "close"],
  ["}", "close"],

  ["#", "dispatch"],
  ["'", "dispatch-quote"],
  ["`", "dispatch-super-quote"],
  ["@", "dispatch-deref"],
  ["~", "dispatch-unquote"],
  ["^", "dispatch-meta"]
];

var separators = '\\s' + escapeRegExp(',;"' + special_chars.map(function(d){
  return d[0];
}).join(''));

var number_regex = (function(){
  var s = '';
  s += '[+-]?';//may start with a + or -
  s += '(' + [
    '([0-9][0-9,]*\\.?[0-9]*)',//starting with a number
    '(\\.[0-9]+)'//starting with a period
  ].join('|') + ')';
  s += '([eE][-+]?[0-9]*)?';//optional exponent (using * b/c the tokenizer needs to match as it goes along)
  return new RegExp('^' + s + '$');
}());

module.exports = function(){
  var t = tokenizer2();

  t.addRule(/^[\s,]+$/, 'whitespace');
  t.addRule(/^;[^\n]*$/, 'comment');
  t.addRule(/(^""$)|(^"([^"]|\\")*[^\\]"$)/, 'string');
  t.addRule(number_regex, 'number');

  special_chars.forEach(function(p){
    t.addRule(new RegExp('^' + escapeRegExp(p[0]) + '$'), p[1]);
  });

  t.addRule(new RegExp('^[^0-9' + separators + '][^' + separators + ']*$'), 'symbol');

  return t;
};
{'fraction_non_alphanumeric': 0.32974220854174685, 'fraction_numerical': 0.01385148133897653, 'mean_word_length': 3.7882136279926337, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6369943', 'n_tokens_mistral': 1071, 'n_tokens_neox': 1046, 'n_words': 176}