<commit_before>var ansiRegEx = require('ansi-regex');

var each = function(obj, fn){
  var key;
  for(key in obj){
    if(obj.hasOwnProperty(key)){
      fn(obj[key], key, obj);
    }
  }
};

var removeElm = function(arr, val){
  var i = arr.indexOf(val);
  if(i >= 0){
    arr.splice(i, 1)
  }
};

//values - first is the code, the reset are escapes
var styles = {
  bold: [1, 21, 22],
  dim: [2, 21, 22],
  italic: [3, 23],
  underline: [4, 24],
  inverse: [7, 27],
  hidden: [8, 28],
  strikethrough: [9, 29],
  black: [30, 39],
  red: [31, 39],
  green: [32, 39],
  yellow: [33, 39],
  blue: [34, 39],
  magenta: [35, 39],
  cyan: [36, 39],
  white: [37, 39],
  gray: [90, 39],
  bgBlack: [40, 49],
  bgRed: [41, 49],
  bgGreen: [42, 49],
  bgYellow: [43, 49],
  bgBlue: [44, 49],
  bgMagenta: [45, 49],
  bgCyan: [46, 49],
  bgWhite: [47, 49]
};

var code_to_style = {};
var escape_codes = {};
var common_escapes = {};

var addEscapeCode = function(escape_code, code, name){<commit_msg>Add support for bright/light variants of colors<commit_after>var ansiRegEx = require('ansi-regex');

var each = function(obj, fn){
  var key;
  for(key in obj){
    if(obj.hasOwnProperty(key)){
      fn(obj[key], key, obj);
    }
  }
};

var removeElm = function(arr, val){
  var i = arr.indexOf(val);
  if(i >= 0){
    arr.splice(i, 1)
  }
};

//values - first is the code, the reset are escapes
var styles = {
  bold: [1, 21, 22],
  dim: [2, 21, 22],
  italic: [3, 23],
  underline: [4, 24],
  inverse: [7, 27],
  hidden: [8, 28],
  strikethrough: [9, 29],
  black: [30, 39],
  red: [31, 39],
  green: [32, 39],
  yellow: [33, 39],
  blue: [34, 39],
  magenta: [35, 39],
  cyan: [36, 39],
  white: [37, 39],
  gray: [90, 39],
  lightRed: [91, 39],
  lightGreen: [92, 39],
  lightYellow: [93, 39],
  lightBlue: [94, 39],
  lightMagenta: [95, 39],
  lightCyan: [96, 39],
  lightWhite: [97, 39],
  bgBlack: [40, 49],
  bgRed: [41, 49],
  bgGreen: [42, 49],
  bgYellow: [43, 49],
  bgBlue: [44, 49],
  bgMagenta: [45, 49],
  bgCyan: [46, 49],
  bgWhite: [47, 49],
  bgGray: [100, 49],
  bgLightRed: [101, 49],
  bgLightGreen: [102, 49],
  bgLightYellow: [103, 49],
  bgLightBlue: [104, 49],
  bgLightMagenta: [105, 49],
  bgLightCyan: [106, 49],
  bgLightWhite: [107, 49],
};

var code_to_style = {};
var escape_codes = {};
var common_escapes = {};

var addEscapeCode = function(escape_code, code, name){
{'fraction_non_alphanumeric': 0.20351023819473466, 'fraction_numerical': 0.10781445883827831, 'mean_word_length': 3.360655737704918, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '840292', 'n_tokens_mistral': 1225, 'n_tokens_neox': 1029, 'n_words': 296}