<commit_before># encoding: UTF-8
$KCODE = 'u' unless "1.9".respond_to?(:encoding)

$:.unshift(File.join(File.dirname(__FILE__), "..", "..", "lib"))
require "htmlentities"
require "benchmark"
require "profiler"

class HTMLEntitiesJob
  def initialize
    @coder = HTMLEntities.new
    @decoded = File.read(File.join(File.dirname(__FILE__), "sample"))
    @encoded = @coder.encode(@decoded, :basic, :named, :hexadecimal)
  end

  def run(cycles)
    cycles.times do
      @coder.encode(@decoded, :basic, :named, :hexadecimal)
      @coder.decode(@encoded)
    end
  end
end

job = HTMLEntitiesJob.new
job.run(100) # Warm up to give JRuby a fair shake.

Benchmark.benchmark do |b|
  b.report{ job.run(100) }
end

Profiler__::start_profile
job.run(1)
Profiler__::print_profile($stdout)<commit_msg>Separated benchmarking and profiling of encoding and decoding to make optimisation easier.<commit_after># encoding: UTF-8
$KCODE = 'u' unless "1.9".respond_to?(:encoding)

$:.unshift(File.join(File.dirname(__FILE__), "..", "..", "lib"))
require "htmlentities"
require "benchmark"
require "profiler"

class HTMLEntitiesJob
  def initialize
    @coder = HTMLEntities.new
    @decoded = File.read(File.join(File.dirname(__FILE__), "sample"))
    @encoded = @coder.encode(@decoded, :basic, :named, :hexadecimal)
  end

  def encode(cycles)
    cycles.times do
      @coder.encode(@decoded, :basic, :named, :hexadecimal)
      @coder.encode(@decoded, :basic, :named, :decimal)
    end
  end

  def decode(cycles)
    cycles.times do
      @coder.decode(@encoded)
    end
  end

  def all(cycles)
    encode(cycles)
    decode(cycles)
  end
end

job = HTMLEntitiesJob.new
job.all(100) # Warm up to give JRuby a fair shake.

Benchmark.benchmark do |b|
  b.report("Encoding"){ job.encode(100) }
  b.report("Decoding"){ job.decode(100) }
end

puts "Encoding"
Profiler__::start_profile
job.encode(1)
Profiler__::print_profile($stdout)

puts "Decoding"
Profiler__::start_profile
job.decode(1)
Profiler__::print_profile($stdout)
{'fraction_non_alphanumeric': 0.14683922349427575, 'fraction_numerical': 0.011946241911398706, 'mean_word_length': 4.521978021978022, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2340368', 'n_tokens_mistral': 757, 'n_tokens_neox': 733, 'n_words': 158}