<commit_before>module Trashed
  module Benchmarker
    def mark!; @mark = now end
    def delta; now - @mark end
    def log!;  Rails.logger.info(delta.to_s) end
  end

  class AdymoMeasurement < Struct.new(:time, :memory, :objects, :gc_runs, :gc_time)
    extend Benchmarker
    FORMAT = 'STATS: %d ms, %.2f KB, %d obj, %d GCs in %d ms'.freeze

    def self.now
      new(Time.now.to_f, GC.allocated_size, ObjectSpace.allocated_objects, GC.collections, GC.time)
    end

    def -(other)
      self.class.new(time - other.time,
        memory - other.memory, objects - other.objects,
        gc_runs - other.gc_runs, gc_time - other.gc_time)
    end

    def to_s
      FORMAT % [time * 1000,
        memory / 1024.0, objects,
        gc_runs, gc_time / 1000.0]
    end
  end

  class LloydMeasurement < Struct.new(:time, :memory, :max_memory, :gc_runs)
    extend Benchmarker
    FORMAT = 'STATS: %d ms, %.2f KB, %.2f KB max, %d GCs'.freeze

    def self.now
      info = GC.heap_info
      new(Time.now.to_f, info['heap_current_memory'], info['heap_max_memory'], info['num_gc_passes'])
    end

    def -(other)
      self.class.new(time - other.time,
        memory - other.memory, max_memory,
        gc_runs - other.gc_runs)
    end

    def to_s
      FORMAT % [time * 1000, memory / 1024.0, max_memory / 1024.0, gc_runs]
    end
  end
end<commit_msg>Treat memory and max memory as samples since they aren't cumulative<commit_after>module Trashed
  module Benchmarker
    def mark!; @mark = now end
    def delta; now - @mark end
    def log!;  Rails.logger.info(delta.to_s) end
  end

  class AdymoMeasurement < Struct.new(:time, :memory, :objects, :gc_runs, :gc_time)
    extend Benchmarker
    FORMAT = 'STATS: %d ms, %.2f KB, %d obj, %d GCs in %d ms'.freeze

    def self.now
      new(Time.now.to_f, GC.allocated_size, ObjectSpace.allocated_objects, GC.collections, GC.time)
    end

    def -(other)
      self.class.new(time - other.time,
        memory - other.memory, objects - other.objects,
        gc_runs - other.gc_runs, gc_time - other.gc_time)
    end

    def to_s
      FORMAT % [time * 1000,
        memory / 1024.0, objects,
        gc_runs, gc_time / 1000.0]
    end
  end

  class LloydMeasurement < Struct.new(:time, :memory, :max_memory, :gc_runs)
    extend Benchmarker
    FORMAT = 'STATS: %d ms, %.2f KB, %.2f KB max, %d GCs'.freeze

    def self.now
      info = GC.heap_info
      new(Time.now.to_f, info['heap_current_memory'], info['heap_max_memory'], info['num_gc_passes'])
    end

    def -(other)
      self.class.new(time - other.time, memory, max_memory, gc_runs - other.gc_runs)
    end

    def to_s
      FORMAT % [time * 1000, memory / 1024.0, max_memory / 1024.0, gc_runs]
    end
  end
end
{'fraction_non_alphanumeric': 0.11938663745892661, 'fraction_numerical': 0.022635998539612998, 'mean_word_length': 2.581699346405229, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5800923', 'n_tokens_mistral': 1087, 'n_tokens_neox': 990, 'n_words': 260}