<commit_before>#!/usr/bin/env ruby

$:.unshift(File.expand_path('../../lib', __FILE__))

require 'bundler/setup'
require 'fluq'
require 'benchmark'
require 'json'

FileUtils.rm_rf FluQ.root.join("log/benchmark")

events = (1..100_000).map do
  FluQ::Event.new "a.b#{rand(4)}.c#{rand(100)}.d#{rand(100)}", Time.now.to_i, "k1" => "value", "k2" => "value", "k3" => "value"
end

handler = FluQ::Handler::Log.new nil,
  path: "log/benchmark/%Y%m/%d/%H/%t.log",
  rewrite: lambda {|t| t.split(".")[1] }

puts "--> Started benchmark"
processed = Benchmark.realtime do
  num = 0
  events.each_slice(100) do |slice|
    handler.on_events(slice)
    num += slice.size
    if (num % 10_000).zero?
      puts "--> Processed #{num}"
    end
  end
end

puts "--> Processed #{events.size} in #{processed.round(1)}s"<commit_msg>Updated benchmark<commit_after>#!/usr/bin/env ruby

$:.unshift(File.expand_path('../../lib', __FILE__))

require 'bundler/setup'
require 'fluq'
require 'benchmark'
require 'json'

FileUtils.rm_rf FluQ.root.join("log/benchmark")

events = (1..100_000).map do
  FluQ::Event.new "a.b#{rand(4)}.c#{rand(100)}.d#{rand(100)}", Time.now.to_i, "k1" => "value", "k2" => "value", "k3" => "value"
end

handler = FluQ::Handler::Log.new nil,
  path: "log/benchmark/%Y%m/%d/%H/%t.log",
  rewrite: lambda {|t| t.split(".")[1] }

puts "--> Started benchmark"
processed = Benchmark.realtime do
  num = 0
  events.each_slice(100) do |slice|
    handler.on_events(slice)
    num += slice.size
    if (num % 10_000).zero?
      puts "--> Processed : #{num}"
    end
  end
  handler = nil
  GC.start
end

puts "--> Processed : #{events.size} in #{processed.round(1)}s"
files   = Dir[FluQ.root.join("log/benchmark/**/*.log").to_s]
lines   = `cat #{files.join(' ')} | wc -l`.strip
puts "--> Written   : #{lines} lines"
{'fraction_non_alphanumeric': 0.22025456557830658, 'fraction_numerical': 0.03099059214167128, 'mean_word_length': 3.8085106382978724, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3907197', 'n_tokens_mistral': 772, 'n_tokens_neox': 728, 'n_words': 142}