<commit_before>package haxed;

import haxed.Common;
import haxed.SyntaxTools;

using haxed.Validate;
using Lambda;
using StringTools;

private enum TToks {
  TDoc;
  TIndent(abs:Int);
  TWhite;
  TString(s:String);
  TKey(k:String);
  TComment;
  TTab;
}

private enum State {
  SDoc;
  SSection;
  SKey;
  SProp;
  SVal;
  SMulti;
  SIndent;
  SError(e:String);
}

class HxpParser {

  var curProp:String;
  var multiIndent:Int;
  var multiVal:StringBuf;
  var keyIndent:Int;
  var hxp:Hxp;
  
  static function
  getTokenizer(r) {
    var
      tk = new Tokenizer<TToks>(r,LINE);    
    
    tk.match(~/\t/,function(re) { return TTab; })
      .match(~/#/,function(re) {return TComment; },Discard)
      .match(~/^\s+(?=\S)/,function(re) {
        return TIndent(re.matchedPos().len);})
      .match(~/^\s+\n/,function (re) { return TWhite; })
      .match(~/^---.*?\n/,function(re) { return TDoc; })
      .match(~/^([a-zA-Z-]+):(?=\s)/,function(re) {
          return TKey(re.matched(1)); })
      .match(~/^([^#]+?)\n/,function(re) {
            return TString(re.matched(1));
        });
     
    return tk; 
  }

  public function new() {}

  function newProperty(p) {
    curProp = p;
    multiVal = new StringBuf();<commit_msg>remove any other references to # in the match<commit_after>package haxed;

import haxed.Common;
import haxed.SyntaxTools;

using haxed.Validate;
using Lambda;
using StringTools;

private enum TToks {
  TDoc;
  TIndent(abs:Int);
  TWhite;
  TString(s:String);
  TKey(k:String);
  TComment;
  TTab;
}

private enum State {
  SDoc;
  SSection;
  SKey;
  SProp;
  SVal;
  SMulti;
  SIndent;
  SError(e:String);
}

class HxpParser {

  var curProp:String;
  var multiIndent:Int;
  var multiVal:StringBuf;
  var keyIndent:Int;
  var hxp:Hxp;
  
  static function
  getTokenizer(r) {
    var
      tk = new Tokenizer<TToks>(r,LINE);    
    
    tk.match(~/\t/,function(re) { return TTab; })
      .match(~/#/,function(re) {return TComment; },Discard)
      .match(~/^\s+(?=\S)/,function(re) {
        return TIndent(re.matchedPos().len);})
      .match(~/^\s+\n/,function (re) { return TWhite; })
      .match(~/^---.*?\n/,function(re) { return TDoc; })
      .match(~/^([a-zA-Z-]+):(?=\s)/,function(re) {
          return TKey(re.matched(1)); })
      .match(~/^(.+?)\n/,function(re) {
            return TString(re.matched(1));
        });
     
    return tk; 
  }

  public function new() {}

  function newProperty(p) {
    curProp = p;
    multiVal = new StringBuf();
{'fraction_non_alphanumeric': 0.1812375249500998, 'fraction_numerical': 0.0015968063872255488, 'mean_word_length': 2.751497005988024, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3784697', 'n_tokens_mistral': 1004, 'n_tokens_neox': 974, 'n_words': 180}