<commit_before>﻿function soramame(source){
  // ソースコード
  this.source = source;

  // 助詞  
  var pat_josi = "[◎|●|は|の|に|で|を|する]";
  this.josi = new RegExp("^" + pat_josi);

  // 文字列リテラル
  var pat_kakko1 = "「.*」|『.*』";
  var pat_kakko2 = "「『";
  this.string_literal = new RegExp("^" + pat_kakko1);
  
  // 数字
  pat_num = "[0-9０-９]";
  this.num = new RegExp("^" + pat_num);
  
  // 記号
  pat_symbol = "[^+-\/%^(){},:＋－×＊÷／％＾（）｛｝、：]";
  this.symbol = new RegExp("^" + pat_symbol);
  
  // 文の終端記号
  this.CRLF = ["\n", "\r", "。"];

  // 単位記号
  this.unit = [
    "メートル", "ドット", "つ目", "つめ", "個目", "個", "円", "つ", "本", "冊", 
    "px", "pt", "cm", "mm", "m", "kg", "g"
  ];

  // その他
  this.yylval;
  this.Escreen = document.getElementById("screen");
  this.debug_clean();
}


soramame.prototype.yylex = function(){
  var retval = WORD;

  // 文の終端を読み飛ばす
  while(this.isCRLF(this.source.charAt(0))){
    this.source = this.source.substring(1);
  }

  // 1文を取り出す
  var line = this.source;
  var l = this.CRLF.length;
  for(var i = 0; i < l; i++){
    var CRLF = this.CRLF[i];
    var p = this.source.indexOf(CRLF);
    if(p > 0){
      line = this.source.substring(0, p);
    }
  }

  // 文字列
  var p = line.match(this.string_literal);
  if (p != null) {
  	var bracket = "'";
    if (p[0].substring(0,1) == "『") {
		bracket = '"';
	}
	var line = bracket + p[0].substring(1, p[0].length - 1) + bracket;
	
	this.source = this.source.substring(p[0].length);
    this.yylval = line + "";
	return STRING;
  }

  // 助詞
  var p = line.match(this.josi);
  if (p != null) {
	var josi = p[0];
	line = line.substring(josi.length);
	this.source = this.source.substring(josi.length);
    this.yylval = josi + "";
	return JOSI;
  }

  // 数字
  if(this.isdigit(line.charAt(0))){
    var token = '';
    var l = line.length;
    for(var i = 0; i < l; i++){
      var c = this.isdigit(line.charAt(i));
      if(!c){
        break;
      }
      token += c;
    }
    this.source = this.source.substring(i);
    this.yylval = token - 0;

    // 単位
    var line = line.substring(i);;
    var l = this.unit.length;
    for(var i = 0; i < l; i++){
      var unit = this.unit[i];
      p = line.indexOf(unit);
      if(p != 0){
        continue;
      }

      // 単位を読み飛ばす
      this.source = this.source.substring(unit.length);
      break;
    }

    return NUMBER;
  }


  // 演算子
  var c = this.isoperator(line.charAt(0));
  if(c){
    this.source = this.source.substring(1);
    this.yylval = c;
    return c.charCodeAt(0);
  }


  // 単語
  var token = line.match(/^[^0123456789+-\/%^(){},:０１２３４５６７８９＋－×＊÷／％＾（）｛｝、：]+/);
  if(token){
    token = String(token);
  } else {
    token = "";
  }
  if(token){
    this.source = this.source.substring(token.length);
  } else {
    token = line;
    this.source = this.source.substring(line.length);
  }
  this.yylval = token;
  if(!token){
    retval = 0;
  }
  return retval;
  /*
  this.debug("line = "+line);<commit_msg>単位を読み飛ばす機能を削除<commit_after>﻿function soramame(source){
  // ソースコード
  this.source = source;

  // 助詞  
  var pat_josi = "[◎|●|は|の|に|で|を|する]";
  this.josi = new RegExp("^" + pat_josi);

  // 文字列リテラル
  var pat_kakko1 = "「.*」|『.*』";
  var pat_kakko2 = "「『";
  this.string_literal = new RegExp("^" + pat_kakko1);
  
  // 数字
  pat_num = "[0-9０-９]";
  this.num = new RegExp("^" + pat_num);
  
  // 記号
  pat_symbol = "[^+-\/%^(){},:＋－×＊÷／％＾（）｛｝、：]";
  this.symbol = new RegExp("^" + pat_symbol);
  
  // 文の終端記号
  this.CRLF = ["\n", "\r", "。"];

  // その他
  this.yylval;
  this.Escreen = document.getElementById("screen");
  this.debug_clean();
}


soramame.prototype.yylex = function(){
  var retval = WORD;

  // 文の終端を読み飛ばす
  while(this.isCRLF(this.source.charAt(0))){
    this.source = this.source.substring(1);
  }

  // 1文を取り出す
  var line = this.source;
  var l = this.CRLF.length;
  for(var i = 0; i < l; i++){
    var CRLF = this.CRLF[i];
    var p = this.source.indexOf(CRLF);
    if(p > 0){
      line = this.source.substring(0, p);
    }
  }

  // 文字列
  var p = line.match(this.string_literal);
  if (p != null) {
  	var bracket = "'";
    if (p[0].substring(0,1) == "『") {
		bracket = '"';
	}
	var line = bracket + p[0].substring(1, p[0].length - 1) + bracket;
	
	this.source = this.source.substring(p[0].length);
    this.yylval = line + "";
	return STRING;
  }

  // 助詞
  var p = line.match(this.josi);
  if (p != null) {
	var josi = p[0];
	line = line.substring(josi.length);
	this.source = this.source.substring(josi.length);
    this.yylval = josi + "";
	return JOSI;
  }

  // 数字
  if(this.isdigit(line.charAt(0))){
    var token = '';
    var l = line.length;
    for(var i = 0; i < l; i++){
      var c = this.isdigit(line.charAt(i));
      if(!c){
        break;
      }
      token += c;
    }
    this.source = this.source.substring(i);
    this.yylval = token - 0;
    return NUMBER;
  }


  // 記号
  var c = this.isoperator(line.charAt(0));
  if(c){
    this.source = this.source.substring(1);
    this.yylval = c;
    return c.charCodeAt(0);
  }


  // 単語
  var token = line.match(/^[^0123456789+-\/%^(){},:０１２３４５６７８９＋－×＊÷／％＾（）｛｝、：]+/);
  if(token){
    token = String(token);
  } else {
    token = "";
  }
  if(token){
    this.source = this.source.substring(token.length);
  } else {
    token = line;
    this.source = this.source.substring(line.length);
  }
  this.yylval = token;
  if(!token){
    retval = 0;
  }
  return retval;
  /*
  this.debug("line = "+line);
{'fraction_non_alphanumeric': 0.21543289643714233, 'fraction_numerical': 0.018460402436773122, 'mean_word_length': 2.4334600760456273, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '593564', 'n_tokens_mistral': 2599, 'n_tokens_neox': 2469, 'n_words': 421}