<commit_before>require 'strscan'

module UTF8Util
  HIGH_BIT_RANGE = /[\x80-\xff]/

  # Check if this String is valid UTF-8
  #
  # Returns true or false.
  def self.valid?(str)
    sc = StringScanner.new(str)

    while sc.skip_until(HIGH_BIT_RANGE)
      sc.pos -= 1

      if !sequence_length(sc)<commit_msg>Add ascii-8bit modifier to regexp<commit_after>require 'strscan'

module UTF8Util
  HIGH_BIT_RANGE = /[\x80-\xff]/n

  # Check if this String is valid UTF-8
  #
  # Returns true or false.
  def self.valid?(str)
    sc = StringScanner.new(str)

    while sc.skip_until(HIGH_BIT_RANGE)
      sc.pos -= 1

      if !sequence_length(sc)
{'fraction_non_alphanumeric': 0.1104199066874028, 'fraction_numerical': 0.017107309486780714, 'mean_word_length': 2.9753086419753085, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7309909', 'n_tokens_mistral': 249, 'n_tokens_neox': 239, 'n_words': 57}