<commit_before># encoding: utf-8

require 'zlib'


module Humboldt
  # @deprecated Use {Rubydoop::JobDescription::secondary_sort}
  class BinaryPrefixPartitioner
    def initialize(cutoff_index)
      @cutoff_index = cutoff_index
    end

    def partition(key, value, num_partitions)
      length = @cutoff_index > key.length ? key.length : @cutoff_index
      prefix = String.from_java_bytes(key.bytes)[0, length]
      Zlib.crc32(prefix) % num_partitions
    end
  end

  # @deprecated Use {Rubydoop::JobDescription::secondary_sort}
  class DropBinaryPrefixPartitioner < BinaryPrefixPartitioner
    def partition(key, value, num_partitions)
      length = key.length > @cutoff_index ? key.length - @cutoff_index : 0
      prefix = String.from_java_bytes(key.bytes)[0, length]
      Zlib.crc32(prefix) % num_partitions
    end
  end

  # @deprecated Use {Rubydoop::JobDescription::secondary_sort}
  class BinaryPrefixComparator
    def initialize(cutoff_index)
      @cutoff_index = cutoff_index
    end

    def compare_raw(bytes1, start1, length1, bytes2, start2, length2)
      subset_length1 = @cutoff_index > length1 ? length1 : @cutoff_index
      subset_length2 = @cutoff_index > length2 ? length2 : @cutoff_index
      ::Hadoop::Io::WritableComparator.compareBytes(bytes1, start1, subset_length1, bytes2, start2, subset_length2)
    end
  end

  # @deprecated Use {Rubydoop::JobDescription::secondary_sort}
  class DropBinaryPrefixComparator < BinaryPrefixComparator
    def compare_raw(bytes1, start1, length1, bytes2, start2, length2)
      subset_length1 = length1 - @cutoff_index
      subset_length2 = length2 - @cutoff_index
      ::Hadoop::Io::WritableComparator.compareBytes(bytes1, start1, subset_length1, bytes2, start2, subset_length2)
    end
  end
end<commit_msg>Fix references in some deprecation warnings<commit_after># encoding: utf-8

require 'zlib'


module Humboldt
  # @deprecated Use {Rubydoop::JobDescription#secondary_sort}
  class BinaryPrefixPartitioner
    def initialize(cutoff_index)
      @cutoff_index = cutoff_index
    end

    def partition(key, value, num_partitions)
      length = @cutoff_index > key.length ? key.length : @cutoff_index
      prefix = String.from_java_bytes(key.bytes)[0, length]
      Zlib.crc32(prefix) % num_partitions
    end
  end

  # @deprecated Use {Rubydoop::JobDescription#secondary_sort}
  class DropBinaryPrefixPartitioner < BinaryPrefixPartitioner
    def partition(key, value, num_partitions)
      length = key.length > @cutoff_index ? key.length - @cutoff_index : 0
      prefix = String.from_java_bytes(key.bytes)[0, length]
      Zlib.crc32(prefix) % num_partitions
    end
  end

  # @deprecated Use {Rubydoop::JobDescription#secondary_sort}
  class BinaryPrefixComparator
    def initialize(cutoff_index)
      @cutoff_index = cutoff_index
    end

    def compare_raw(bytes1, start1, length1, bytes2, start2, length2)
      subset_length1 = @cutoff_index > length1 ? length1 : @cutoff_index
      subset_length2 = @cutoff_index > length2 ? length2 : @cutoff_index
      ::Hadoop::Io::WritableComparator.compareBytes(bytes1, start1, subset_length1, bytes2, start2, subset_length2)
    end
  end

  # @deprecated Use {Rubydoop::JobDescription#secondary_sort}
  class DropBinaryPrefixComparator < BinaryPrefixComparator
    def compare_raw(bytes1, start1, length1, bytes2, start2, length2)
      subset_length1 = length1 - @cutoff_index
      subset_length2 = length2 - @cutoff_index
      ::Hadoop::Io::WritableComparator.compareBytes(bytes1, start1, subset_length1, bytes2, start2, subset_length2)
    end
  end
end
{'fraction_non_alphanumeric': 0.08727070594774874, 'fraction_numerical': 0.023346303501945526, 'mean_word_length': 4.387724550898204, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3198887', 'n_tokens_mistral': 1180, 'n_tokens_neox': 1146, 'n_words': 242}