<commit_before>var SNIP_STR = "...";
var COL_CONTENT_LENGTH_MAX_DEFAULT = 32;

function puts(){
  console.log.apply(console, arguments);
}

function escapeHtml(s){
  return s
      .replace(/&/g, "&amp;")
      .replace(/</g, "&lt;")
      .replace(/>/g, "&gt;")
      .replace(/"/g, "&quot;")
      .replace(/'/g, "&#39;");
}

function strip(s){
  return s.replace(/^\s+|\s+$/g, "");
}

/**
 * http://winter-tail.sakura.ne.jp/pukiwiki/index.php?JavaScript%A4%A2%A4%EC%A4%B3%A4%EC%2F%C0%B5%B5%AC%C9%BD%B8%BD%A5%D1%A5%BF%A1%BC%A5%F3
 */
function includeZenkaku(s){
  return /[^ -~｡-ﾟ]/.test(s);
}

function strlen(s){
  if( includeZenkaku(s) ){
    var len = 0;
    for(var i=0,slen=s.length; i<slen; i++){
      if( includeZenkaku(s.charAt(i)) ){
        len += 2;
      }else{
        len += 1;
      }
    }
    return len;
  }else{
    return s.length;
  }
}

function mkstr(s, n){
  var ret = "";
  for(var i=0; i<n; i++){
    ret += s;
  }
  return ret;
}

function padLeft(s, n){
  var pad = n - strlen(s);
  return s + mkstr(" ", pad);
}

function padRight(s, n){
  var pad = n - strlen(s);
  return mkstr(" ", pad) + s;
}

function isNumber(s){
  return /^-?[\d,]+$/.test(s);
}

function parse_regexp(text, options){
  var lines = text.split("\n");
  var re = options.re;
  return _.chain(lines).filter(function(line){
    return ! /^\s*$/.test(line);
  }).map(function(line){
    return line.split(re);
  }).value();
}

function parse_mysql(text){
  var lines = text.split("\n");
  return _.chain(lines).filter(function(line){
    return ! ( /^\+/.test(line)
               || /^\s*$/.test(line)
             );
  }).map(function(line){
    var cols = (" " + line + " ").split(" | ");
    cols.shift();
    cols.pop();
    return cols.map(strip);
  }).value();
}

function parse_postgresql(text){
  var lines = text.split("\n");<commit_msg>add StrScan<commit_after>var SNIP_STR = "...";
var COL_CONTENT_LENGTH_MAX_DEFAULT = 32;

function puts(){
  console.log.apply(console, arguments);
}

function escapeHtml(s){
  return s
      .replace(/&/g, "&amp;")
      .replace(/</g, "&lt;")
      .replace(/>/g, "&gt;")
      .replace(/"/g, "&quot;")
      .replace(/'/g, "&#39;");
}

function strip(s){
  return s.replace(/^\s+|\s+$/g, "");
}

/**
 * http://winter-tail.sakura.ne.jp/pukiwiki/index.php?JavaScript%A4%A2%A4%EC%A4%B3%A4%EC%2F%C0%B5%B5%AC%C9%BD%B8%BD%A5%D1%A5%BF%A1%BC%A5%F3
 */
function includeZenkaku(s){
  return /[^ -~｡-ﾟ]/.test(s);
}

function strlen(s){
  if( includeZenkaku(s) ){
    var len = 0;
    for(var i=0,slen=s.length; i<slen; i++){
      if( includeZenkaku(s.charAt(i)) ){
        len += 2;
      }else{
        len += 1;
      }
    }
    return len;
  }else{
    return s.length;
  }
}

function mkstr(s, n){
  var ret = "";
  for(var i=0; i<n; i++){
    ret += s;
  }
  return ret;
}

function padLeft(s, n){
  var pad = n - strlen(s);
  return s + mkstr(" ", pad);
}

function padRight(s, n){
  var pad = n - strlen(s);
  return mkstr(" ", pad) + s;
}

function isNumber(s){
  return /^-?[\d,]+$/.test(s);
}


var StrScan = (function(){

  function StrScan(str){
    this.str = str;
    this.rest = this.str;
    this.pos = 0;
    this.posBom = 0; // beginning of match
    this.m = null; // match result
  }
  var __ = StrScan.prototype;

  __.scan = function(re){
    this.posBom = this.pos;

    if( ! re.test(this.rest) ){
      this.m = null;
      return null;
    }

    var ret = this.rest.match(re);
    this.m = ret;
    this.movePos(ret[0].length);

    return true;
  };

  /**
   * EOS: end of string
   */
  __.isEos = function(){
    return this.pos >= this.str.length;
  };

  __.substring = function(i0, i1){
    return this.str.substring(i0, i1);
  };

  __.movePos = function(delta){
    this.pos += delta;
    this.rest = this.str.substring(this.pos);
  };

  return StrScan;
})();


function parse_regexp(text, options){
  var lines = text.split("\n");
  var re = options.re;
  return _.chain(lines).filter(function(line){
    return ! /^\s*$/.test(line);
  }).map(function(line){
    return line.split(re);
  }).value();
}

function parse_mysql(text){
  var lines = text.split("\n");
  return _.chain(lines).filter(function(line){
    return ! ( /^\+/.test(line)
               || /^\s*$/.test(line)
             );
  }).map(function(line){
    var cols = (" " + line + " ").split(" | ");
    cols.shift();
    cols.pop();
    return cols.map(strip);
  }).value();
}

function parse_postgresql(text){
  var lines = text.split("\n");
{'fraction_non_alphanumeric': 0.22177688710754842, 'fraction_numerical': 0.013582720997550656, 'mean_word_length': 2.6729354047424367, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5569666', 'n_tokens_mistral': 1879, 'n_tokens_neox': 1832, 'n_words': 335}