<commit_before>class Hash
  # Treat the array as a hash of lists. This method will append 'value' to the list
  # at key 'key' in the hash. If there is no list for 'key', one is created.
  def pushToList(key, value)
    list = self[key]
    if ! list
      list = [] 
      self[key] = list
    end
    list.push value
  end
end

module QuartzTorrent
  # This is Linux specific: system call number for gettid
  SYSCALL_GETTID = 224

  def self.bytesToHex(v, addSpaces = nil)
    s = ""
    v.each_byte{ |b|
      hex = b.to_s(16)
      hex = "0" + hex if hex.length == 1
      s << hex
      s << " " if addSpaces == :add_spaces
    }
    s
  end

  def self.arrayShuffleRange!(array, start, length)
    raise "Invalid range" if start + length > array.size

    (start+length).downto(start+1) do |i|
      r = start + rand(i-start)
      array[r], array[i-1] = array[i-1], array[r]
    end
    true
  end<commit_msg>Added hexToBytes method as the inverse of bytesToHex<commit_after>class Hash
  # Treat the array as a hash of lists. This method will append 'value' to the list
  # at key 'key' in the hash. If there is no list for 'key', one is created.
  def pushToList(key, value)
    list = self[key]
    if ! list
      list = [] 
      self[key] = list
    end
    list.push value
  end
end

module QuartzTorrent
  # This is Linux specific: system call number for gettid
  SYSCALL_GETTID = 224

  def self.bytesToHex(v, addSpaces = nil)
    s = ""
    v.each_byte{ |b|
      hex = b.to_s(16)
      hex = "0" + hex if hex.length == 1
      s << hex
      s << " " if addSpaces == :add_spaces
    }
    s
  end

  def self.hexToBytes(v)
    [v].pack "H*"
  end

  def self.arrayShuffleRange!(array, start, length)
    raise "Invalid range" if start + length > array.size

    (start+length).downto(start+1) do |i|
      r = start + rand(i-start)
      array[r], array[i-1] = array[i-1], array[r]
    end
    true
  end
{'fraction_non_alphanumeric': 0.11192087454450807, 'fraction_numerical': 0.010411244143675169, 'mean_word_length': 2.4078014184397163, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2613818', 'n_tokens_mistral': 706, 'n_tokens_neox': 678, 'n_words': 229}