<commit_before>const SNIP_STR = "...";
const NULL_STR = "NULL";
const COL_CONTENT_LENGTH_MAX_DEFAULT = 32;
const NUM_ROWS_MAX = 500;

function puts(){
  console.log.apply(console, arguments);
}

function strip(s){
  return s.replace(/^\s+|\s+$/g, "");
}

/**
 * http://winter-tail.sakura.ne.jp/pukiwiki/index.php?JavaScript%A4%A2%A4%EC%A4%B3%A4%EC%2F%C0%B5%B5%AC%C9%BD%B8%BD%A5%D1%A5%BF%A1%BC%A5%F3
 */
function includeZenkaku(s){
  return /[^ -~｡-ﾟ]/.test(s);
}

function strlen(s){
  if(s == null){
    return 0;
  }
  if( includeZenkaku(s) ){
    var len = 0;
    for(let i=0,slen=s.length; i<slen; i++){
      if( includeZenkaku(s.charAt(i)) ){
        len += 2;
      }else{
        len += 1;
      }
    }
    return len;
  }else{
    return s.length;
  }
}

function mkstr(s, n){
  var ret = "";
  for(let i=0; i<n; i++){
    ret += s;
  }
  return ret;
}

function padRight(s, n){
  var pad = n - strlen(s);
  return s + mkstr(" ", pad);
}

function padLeft(s, n){
  var pad = n - strlen(s);
  return mkstr(" ", pad) + s;
}

function mapChars(str, fn){
  var chars = [];
  for(let i=0,len=str.length; i<len; i++){
    chars.push(fn(str.charAt(i), i));
  }
  return chars;
}

function isNumber(s){
  return /^-?[\d,]+$/.test(s);
}

function mkSpanHtml(content, className){
  return '<span class="' + className + '">' + content + '</span>';
}

function sqlEscape(str){
  return str
    .replace(/\\/g, "\\\\")
    .replace(/'/g, "\\'")
    .replace(/\r/g, "\\r")
    .replace(/\n/g, "\\n")
    .replace(/\t/g, "\\t")
  ;
}


const SPAN_WS = mkSpanHtml(" ", "col_space");

const SPAN_CTRL_CD_MAP = {
  "\\": mkSpanHtml("\\\\", "col_ctrl_cd"),
  "\r": mkSpanHtml("\\r", "col_ctrl_cd"),
  "\n": mkSpanHtml("\\n", "col_ctrl_cd"),
  "\t": mkSpanHtml("\\t", "col_ctrl_cd")
};


var StrScan = (function(){

  function StrScan(str){
    this.str = str;
    this.rest = this.str;
    this.pos = 0;
    this.posBom = 0; // beginning of match
    this.m = null; // match result
  }
  var __ = StrScan.prototype;

  __.scan = function(re){
    this.posBom = this.pos;

    if( ! re.test(this.rest) ){
      this.m = null;
      return null;
    }

    var ret = this.rest.match(re);
    this.m = ret;
    this.movePos(ret[0].length);

    return true;
  };

  /**
   * EOS: end of string
   */
  __.isEos = function(){
    return this.pos >= this.str.length;
  };

  __.substring = function(i0, i1){
    return this.str.substring(i0, i1);
  };

  __.movePos = function(delta){
    this.pos += delta;
    this.rest = this.str.substring(this.pos);
  };

  return StrScan;
})();


var ColContent = {
  _tokenize: function(str){
    // create token
    function _t(type, str){
      return { type: type, str: str };
    }

    var ts = [];
    var posPrevEom = 0; // previous end of match
    var ss = new StrScan(str);

    /**
     * \u0020: normal white space
     * \u00a0: nbsp
     */
    while( ! ss.isEos() ){
      if( ss.scan( /^[\u0020\u00a0]+/ ) ){
        if( posPrevEom < ss.posBom ){
          ts.push( _t("plain", ss.substring(posPrevEom, ss.posBom)) );
        }
        ts.push( _t("space", ss.m[0]) );
        posPrevEom = ss.pos;
      }else if( ss.scan(/^[\t\r\n\\]/) ){
        if( posPrevEom < ss.posBom ){
          ts.push( _t("plain", ss.substring(posPrevEom, ss.posBom)) );
        }
        ts.push( _t("ctrl_cd", ss.m[0]) );
        posPrevEom = ss.pos;
      }else{<commit_msg>refactor: class StrScan<commit_after>const SNIP_STR = "...";
const NULL_STR = "NULL";
const COL_CONTENT_LENGTH_MAX_DEFAULT = 32;
const NUM_ROWS_MAX = 500;

function puts(){
  console.log.apply(console, arguments);
}

function strip(s){
  return s.replace(/^\s+|\s+$/g, "");
}

/**
 * http://winter-tail.sakura.ne.jp/pukiwiki/index.php?JavaScript%A4%A2%A4%EC%A4%B3%A4%EC%2F%C0%B5%B5%AC%C9%BD%B8%BD%A5%D1%A5%BF%A1%BC%A5%F3
 */
function includeZenkaku(s){
  return /[^ -~｡-ﾟ]/.test(s);
}

function strlen(s){
  if(s == null){
    return 0;
  }
  if( includeZenkaku(s) ){
    var len = 0;
    for(let i=0,slen=s.length; i<slen; i++){
      if( includeZenkaku(s.charAt(i)) ){
        len += 2;
      }else{
        len += 1;
      }
    }
    return len;
  }else{
    return s.length;
  }
}

function mkstr(s, n){
  var ret = "";
  for(let i=0; i<n; i++){
    ret += s;
  }
  return ret;
}

function padRight(s, n){
  var pad = n - strlen(s);
  return s + mkstr(" ", pad);
}

function padLeft(s, n){
  var pad = n - strlen(s);
  return mkstr(" ", pad) + s;
}

function mapChars(str, fn){
  var chars = [];
  for(let i=0,len=str.length; i<len; i++){
    chars.push(fn(str.charAt(i), i));
  }
  return chars;
}

function isNumber(s){
  return /^-?[\d,]+$/.test(s);
}

function mkSpanHtml(content, className){
  return '<span class="' + className + '">' + content + '</span>';
}

function sqlEscape(str){
  return str
    .replace(/\\/g, "\\\\")
    .replace(/'/g, "\\'")
    .replace(/\r/g, "\\r")
    .replace(/\n/g, "\\n")
    .replace(/\t/g, "\\t")
  ;
}


const SPAN_WS = mkSpanHtml(" ", "col_space");

const SPAN_CTRL_CD_MAP = {
  "\\": mkSpanHtml("\\\\", "col_ctrl_cd"),
  "\r": mkSpanHtml("\\r", "col_ctrl_cd"),
  "\n": mkSpanHtml("\\n", "col_ctrl_cd"),
  "\t": mkSpanHtml("\\t", "col_ctrl_cd")
};


class StrScan {

  constructor(str){
    this.str = str;
    this.rest = this.str;
    this.pos = 0;
    this.posBom = 0; // beginning of match
    this.m = null; // match result
  }

  scan(re){
    this.posBom = this.pos;

    if( ! re.test(this.rest) ){
      this.m = null;
      return null;
    }

    var ret = this.rest.match(re);
    this.m = ret;
    this.movePos(ret[0].length);

    return true;
  }

  /**
   * EOS: end of string
   */
  isEos(){
    return this.pos >= this.str.length;
  }

  substring(i0, i1){
    return this.str.substring(i0, i1);
  }

  movePos(delta){
    this.pos += delta;
    this.rest = this.str.substring(this.pos);
  }
}


var ColContent = {
  _tokenize: function(str){
    // create token
    function _t(type, str){
      return { type: type, str: str };
    }

    var ts = [];
    var posPrevEom = 0; // previous end of match
    var ss = new StrScan(str);

    /**
     * \u0020: normal white space
     * \u00a0: nbsp
     */
    while( ! ss.isEos() ){
      if( ss.scan( /^[\u0020\u00a0]+/ ) ){
        if( posPrevEom < ss.posBom ){
          ts.push( _t("plain", ss.substring(posPrevEom, ss.posBom)) );
        }
        ts.push( _t("space", ss.m[0]) );
        posPrevEom = ss.pos;
      }else if( ss.scan(/^[\t\r\n\\]/) ){
        if( posPrevEom < ss.posBom ){
          ts.push( _t("plain", ss.substring(posPrevEom, ss.posBom)) );
        }
        ts.push( _t("ctrl_cd", ss.m[0]) );
        posPrevEom = ss.pos;
      }else{
{'fraction_non_alphanumeric': 0.210187265917603, 'fraction_numerical': 0.016179775280898877, 'mean_word_length': 2.404385517593065, 'pattern_counts': {'":': 8, '<': 17, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4270558', 'n_tokens_mistral': 2860, 'n_tokens_neox': 2749, 'n_words': 531}