<commit_before>dir, definitions = ARGV

if defined? Encoding
  def readlines(name)
    IO.readlines(name, :encoding => "ascii-8bit")
  end
else
  def readlines(name)
    IO.readlines(name)
  end
end

# This sort logic is ctitical for determining the encoding index, which is
# internally used in String#<=>.
# It must be IDENTICAL to MRI's counterpart logic.

def sort_by_encoding_index(files)
  files.sort_by do |file|
    token_pairs = file.scan(/(\D+)|(\d+)/)

    sort_key = token_pairs.map do |letters, digits|
      if letters
        letters
      else
        padded_numeric_sort_key(digits)
      end
    end

    sort_key.flatten
  end
end
<commit_msg>Fix typo<commit_after>dir, definitions = ARGV

if defined? Encoding
  def readlines(name)
    IO.readlines(name, :encoding => "ascii-8bit")
  end
else
  def readlines(name)
    IO.readlines(name)
  end
end

# This sort logic is critical for determining the encoding index, which is
# internally used in String#<=>.
# It must be IDENTICAL to MRI's counterpart logic.

def sort_by_encoding_index(files)
  files.sort_by do |file|
    token_pairs = file.scan(/(\D+)|(\d+)/)

    sort_key = token_pairs.map do |letters, digits|
      if letters
        letters
      else
        padded_numeric_sort_key(digits)
      end
    end

    sort_key.flatten
  end
end

{'fraction_non_alphanumeric': 0.09401061410159212, 'fraction_numerical': 0.001516300227445034, 'mean_word_length': 3.125, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7564909', 'n_tokens_mistral': 465, 'n_tokens_neox': 441, 'n_words': 129}