<commit_before>require "grok"
require "pp"

patterns = {}

matches = [
  #"%{SYSLOGBASE} Accepted %{NOTSPACE:method} for %{DATA:user} from %{IPORHOST:client} port %{INT:port}",
  #"%{SYSLOGBASE} Did not receive identification string from %{IPORHOST:client}",
  #"%{SYSLOGBASE} error: PAM: authentication error for %{DATA:user} from %{IPORHOST:client}",
  "%{SYSLOGBASE} .*"
  #"%{COMBINEDAPACHELOG}",
  #"%{UNINDEXED}hello (?=%{GREEDYDATA})%{WORD}"
  
  #"( *%{DATA:key}:%{NOTSPACE:value})+"
]

pile = Grok::Pile.new
pile.add_patterns_from_file("../patterns/base")
matches.collect do |m|
  #g = Grok.new
  #g.add_patterns_from_file("../patterns/base")
  pile.compile(m)
end

bytes = 0
time_start = Time.now.to_f
$stdin.each do |line|
  grok, m = pile.match(line)
  if m
    #data = Hash.new { |h,k| h[k] = Array.new }
    #m.each_capture do |key, value|
      #data[key] << value
    #end
    #pp data
    #pp m.captures
    m.each_capture do |key, value|
      p key => value
    end

    #bytes += line.length
    break
  end
end

#time_end = Time.now.to_f
#puts "parse rate: #{ (bytes / 1024) / (time_end - time_start) }"<commit_msg>- clean up the sample<commit_after>$: << File.join(File.dirname(__FILE__), "lib")
require "grok-pure"

patterns = {}

matches = [
  #"%{SYSLOGBASE} Accepted %{NOTSPACE:method} for %{DATA:user} from %{IPORHOST:client} port %{INT:port}",
  #"%{SYSLOGBASE} Did not receive identification string from %{IPORHOST:client}",
  #"%{SYSLOGBASE} error: PAM: authentication error for %{DATA:user} from %{IPORHOST:client}",
  #"%{QUOTEDSTRING}"
  "%{COMBINEDAPACHELOG}",
  #"%{UNINDEXED}hello (?=%{GREEDYDATA})%{WORD}"
  
  #"( *%{DATA:key}:%{NOTSPACE:value})+"
]

pile = Grok::Pile.new
pile.add_patterns_from_file("patterns/pure-ruby/base")
matches.collect do |m|
  pile.compile(m)
end

bytes = 0
time_start = Time.now.to_f
$stdin.each do |line|
  grok, m = pile.match(line)
  if m
    m.each_capture do |key, value|
      p key => value
    end

    #bytes += line.length
    break
  end
end

#time_end = Time.now.to_f
#puts "parse rate: #{ (bytes / 1024) / (time_end - time_start) }"
{'fraction_non_alphanumeric': 0.1843601895734597, 'fraction_numerical': 0.004739336492890996, 'mean_word_length': 4.11138014527845, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '295174', 'n_tokens_mistral': 863, 'n_tokens_neox': 846, 'n_words': 182}