<commit_before>class RHeap
  def initialize data=[], &compare_by
    @size = data.size
    if compare_by
      @table = {}
      @compare_by = compare_by
      @heap = data.map(&compare_by).uniq.sort
      data.each do |value|
        key = compare_by[value]
        (@table[key] ||= []) << value
      end
    else
      @heap = data.sort
    end
  end
  def enq data
    @size += 1
    if @compare_by
      key = @compare_by[data]
      if @table[key]
        @table[key] << data
        return self
      end
      @table[key] = [data]
      value = key
    else
      value = data
    end
    index = @heap.size
    while index > 0
      pindex = (index-1)/2
      pvalue = @heap[pindex]
      break if pvalue < value
      @heap[index] = pvalue
      index = pindex
    end
    @heap[index] = value
    self
  end
  def first
    value = @heap.first
    return value unless @compare_by
    @table[value].first if value
  end
  def deq
    return nil if @heap.empty?
    @size -= 1
    index = 0
    value = @heap[index]
    if @compare_by
      list = @table[value]
      data = list.shift
      return data unless list.empty?
      @table.delete value
    else
      data = value
    end
    value = @heap.pop
    return data if @heap.empty?
    while true
      lindex, rindex = 2*index+1, 2*index+2
      left, right = @heap[lindex], @heap[rindex]
      break unless left
      cindex, cvalue = !right || left < right ? [lindex, left] : [rindex, right]
      break unless value > cvalue
      @heap[index] = cvalue
      index = cindex
    end
    @heap[index] = value
    data
  end
  def to_s;"#<#{self.class.name}:0x#{(object_id*2).to_s(16)}[#{@size}]>";end
  def empty?;@heap.empty?;end
  def size;@size;end
  alias pop deq
  alias shift deq
  alias push enq
  alias unshift enq
  alias << enq
  alias inspect to_s
end<commit_msg>faster<commit_after>class RHeap
  def initialize data=[], &compare_by
    @size = data.size
    if compare_by
      @table = {}
      @compare_by = compare_by
      @heap = data.map(&compare_by).uniq.sort
      data.each do |value|
        key = compare_by[value]
        (@table[key] ||= []) << value
      end
    else
      @heap = data.sort
    end
  end
  def enq data
    @size += 1
    if @compare_by
      key = @compare_by[data]
      if @table[key]
        @table[key] << data
        return self
      end
      @table[key] = [data]
      value = key
    else
      value = data
    end
    index = @heap.size
    while index > 0
      pindex = (index-1)/2
      pvalue = @heap[pindex]
      break if pvalue < value
      @heap[index] = pvalue
      index = pindex
    end
    @heap[index] = value
    self
  end
  def first
    value = @heap.first
    return value unless @compare_by
    @table[value].first if value
  end
  def deq
    return nil if @heap.empty?
    @size -= 1
    value = @heap.first
    if @compare_by
      list = @table[value]
      data = list.shift
      return data unless list.empty?
      @table.delete value
    else
      data = value
    end
    value = @heap.pop
    lastindex = @heap.size-1
    return data if @heap.empty?
    index = 0
    heap = @heap
    while true
      lindex = 2*index+1
      break if lindex > lastindex
      lvalue = heap[lindex]
      if lindex < lastindex
        rvalue = heap[lindex+1]
        unless lvalue < rvalue
          lindex += 1
          lvalue = rvalue
        end
      end
      break unless value > lvalue
      heap[index] = lvalue
      index = lindex
    end
    heap[index] = value
    data
  end
  def to_s;"#<#{self.class.name}:0x#{(object_id*2).to_s(16)}[#{@size}]>";end
  def empty?;@heap.empty?;end
  def size;@size;end
  alias pop deq
  alias shift deq
  alias push enq
  alias unshift enq
  alias << enq
  alias inspect to_s
end
{'fraction_non_alphanumeric': 0.10795755968169761, 'fraction_numerical': 0.007692307692307693, 'mean_word_length': 1.966955153422502, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2700792', 'n_tokens_mistral': 1300, 'n_tokens_neox': 1294, 'n_words': 363}