<commit_before>function Definition(pattern, term) {
  this.regex = new RegExp("^" + pattern)
  this.term = term
}
function Terminologic() {
  var self = this
  self.startDefinition = null
  self.endDefinition = null
  self.definitions = []
  self.substitutes = []
  self.restitution = null
  self.add = function(pattern, term) {
    self.definitions.push(new Definition(pattern, term))
    return self;
  }
}
function Token(text, term) {
  this.text = text
  this.term = term
}
function getDefaultSettings() {
  var settings = {}
  settings.rootTerminologic = new Terminologic()
  var root = function(pattern, term) {
    settings.rootTerminologic.add(pattern, term)
  }
  root("\\s+", "blank")
  root("\\w+", "id")
  root(".", "unknown")
  var lineCommentTerminologic = new Terminologic()
  lineCommentTerminologic.startDefinition = new Definition("//", "lc-start")
  lineCommentTerminologic.endDefinition = new Definition("(\\r\\n|\\r|\\n)", "lc-end")
  lineCommentTerminologic.add("[^\\r\\n]+", "lc-body")
  lineCommentTerminologic.restitution = settings.rootTerminologic
  settings.rootTerminologic.substitutes.push(lineCommentTerminologic)
  var blockCommentTerminologic = new Terminologic()
  blockCommentTerminologic.startDefinition = new Definition("/\\*", "bc-start")
  blockCommentTerminologic.endDefinition = new Definition("\\*/", "bc-end")
  blockCommentTerminologic.add("((?!\\*/).)+", "bc-body")
  blockCommentTerminologic.restitution = settings.rootTerminologic
  settings.rootTerminologic.substitutes.push(blockCommentTerminologic)
  return settings;
}
function Tokenizer(settings) {
  var self = this, text, nextIndex, terminologic
  self.init = function () {
    text = ""
    nextIndex = 0<commit_msg>Clean getDefaultSettings up<commit_after>function Definition(pattern, term) {
  this.regex = new RegExp("^" + pattern)
  this.term = term
}
function Terminologic() {
  var self = this
  self.startDefinition = null
  self.endDefinition = null
  self.definitions = []
  self.substitutes = []
  self.restitution = null
  self.add = function(pattern, term) {
    self.definitions.push(new Definition(pattern, term))
    return self;
  }
}
function Token(text, term) {
  this.text = text
  this.term = term
}
function getDefaultSettings() {
  var settings = {}
  var root = new Terminologic()
  settings.rootTerminologic = root
  root.add("\\s+", "blank")
  root.add("\\w+", "id")
  root.add(".", "unknown")
  // Line comment
  var lc = new Terminologic()
  lc.startDefinition = new Definition("//", "lc-start")
  lc.endDefinition = new Definition("(\\r\\n|\\r|\\n)", "lc-end")
  lc.add("[^\\r\\n]+", "lc-body")
  lc.restitution = root
  root.substitutes.push(lc)
  // Block comment
  var bc = new Terminologic()
  bc.startDefinition = new Definition("/\\*", "bc-start")
  bc.endDefinition = new Definition("\\*/", "bc-end")
  bc.add("((?!\\*/).)+", "bc-body")
  bc.restitution = root
  root.substitutes.push(bc)
  return settings;
}
function Tokenizer(settings) {
  var self = this, text, nextIndex, terminologic
  self.init = function () {
    text = ""
    nextIndex = 0
{'fraction_non_alphanumeric': 0.1595985755908061, 'fraction_numerical': 0.0006474587245063128, 'mean_word_length': 4.040783034257749, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6995364', 'n_tokens_mistral': 1041, 'n_tokens_neox': 1001, 'n_words': 212}