<commit_before><commit_msg>add tag-attr-lexer.js<commit_after>var TagAttrLexer = (function(){
  var __rex_symbol = /[^=\s]+/;

  // lexer src is attribute parts of original tag source.
  // so if tag source is "<div class='nehan-float-start'>",
  // then lexer src is "class='nehan-float-start'".
  function TagAttrLexer(src){
    this.buff = src;
  }

  TagAttrLexer.prototype = {
    isEnd : function(){
      return this.buff === "";
    },
    get : function(){
      var c1 = this._peek();
      if(c1 === null){
	return null;
      }
      switch(c1){
      case "=":
	this._step(1);
	return c1;
      case "'": case "\"":
	return this._getLiteral(c1);
      case " ":
	this._step(1);
	return this.get(); // skip space
      default:
	return this._getSymbol();
      }
    },
    _peek : function(){
      return this.buff? this.buff.charAt(0) : null;
    },
    _step : function(length){
      this.buff = this.buff.substring(length);
    },
    _getSymbol : function(){
      var symbol = HtmlLexer.prototype._getByRex.call(this, __rex_symbol);
      if(symbol){
	this._step(symbol.length);
      }
      return symbol;
    },
    _getLiteral : function(quote_char){
      var quote_end_pos = this.buff.indexOf(quote_char, 1);
      if(quote_end_pos < 0){
	throw "TagAttrLexer::syntax error(literal not closed)";
      }
      var literal = this.buff.substring(1, quote_end_pos);
      this._step(quote_end_pos + 1);
      return literal;
    }
  };

  return TagAttrLexer;
})();

{'fraction_non_alphanumeric': 0.15524193548387097, 'fraction_numerical': 0.008064516129032258, 'mean_word_length': 2.6140776699029127, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2447709', 'n_tokens_mistral': 528, 'n_tokens_neox': 509, 'n_words': 107}