<commit_before># frozen_string_literal: true

require 'uri'

class Groupie
  # Tokenizer helps turn a String into an Array of Strings that are the
  # individual tokens (mostly words) from the input.
  #
  # Please consider this entire class to be a private API,
  # and use Groupie.tokenize to tokenize things.
  class Tokenizer
    def initialize(input)
      # Ensure our input is converted to a String and duplicated so we can modify it in-place
      @raw = input.to_s.dup
    end

    def to_tokens
      return @tokens if @tokens

      # Ignore case by downcasing everything
      @raw.downcase!
      # Convert all types of whitespace (space, tab, newline) into regular spaces
      @raw.gsub!(/\s+/, ' ')
      # Strip HTML tags entirely
      @raw.gsub!(/<[^>]+?>/, ' ')
      # Intelligently split URLs into their component parts
      @raw.gsub!(%r{http[\w\-\#:/_.?&=]+}) do |url|
        uri = URI.parse(url)
        path = uri.path.to_s
        path.tr!('/_\-', ' ')
        query = uri.query.to_s
        query.tr!('?=&#_\-', ' ')
        fragment = uri.fragment.to_s
        fragment.tr!('#_/\-', ' ')
        "#{uri.scheme} #{uri.host} #{path} #{query} #{fragment}"
      end
      # Strip characters not likely to be part of a word or number
      @raw.gsub!(/[^\w\ \-.,]/, ' ')
      # Split the resulting string on whitespace
      @tokens = @raw.split.map do |str|
        # Final cleanup of wrapped quotes and interpunction
        str.gsub!(/\A['"]+|[!,."']+\Z/, '')
        str
      end

      @tokens
    end
  end
end<commit_msg>Refactor: split Tokenizer into helper methods<commit_after># frozen_string_literal: true

require 'uri'

class Groupie
  # Tokenizer helps turn a String into an Array of Strings that are the
  # individual tokens (mostly words) from the input.
  #
  # Please consider this entire class to be a private API,
  # and use Groupie.tokenize to tokenize things.
  class Tokenizer
    def initialize(input)
      # Ensure our input is converted to a String and duplicated so we can modify it in-place
      @raw = input.to_s.dup
    end

    def to_tokens
      return @tokens if @tokens

      # In-place modifications to our @raw String
      downcase!
      normalize_whitespace!
      strip_html_tags!
      tokenize_urls!
      strip_non_word_characters!

      # Split the resulting string on whitespace and clean up the token candidates
      @tokens = @raw.split.map { |str| remove_interpunction!(str) }

      @tokens
    end

    private

    # Ignore case by downcasing everything
    def downcase!
      @raw.downcase!
    end

    # Convert all types of whitespace (space, tab, newline) into regular spaces
    def normalize_whitespace!
      @raw.gsub!(/\s+/, ' ')
    end

    # Strip HTML tags entirely
    def strip_html_tags!
      @raw.gsub!(/<[^>]+?>/, ' ')
    end

    # Intelligently split URLs into their component parts
    def tokenize_urls!
      @raw.gsub!(%r{http[\w\-\#:/_.?&=]+}) do |url|
        uri = URI.parse(url)
        path = uri.path.to_s
        path.tr!('/_\-', ' ')
        query = uri.query.to_s
        query.tr!('?=&#_\-', ' ')
        fragment = uri.fragment.to_s
        fragment.tr!('#_/\-', ' ')
        "#{uri.scheme} #{uri.host} #{path} #{query} #{fragment}"
      end
    end

    # Strip characters not likely to be part of a word or number
    def strip_non_word_characters!
      @raw.gsub!(/[^\w\ \-.,]/, ' ')
    end

    # Remove wrapping quotes and interpunction from individual token candidates
    def remove_interpunction!(str)
      str.gsub!(/\A['"]+|[!,."']+\Z/, '')
      str
    end
  end
end
{'fraction_non_alphanumeric': 0.12801330008312553, 'fraction_numerical': 0.0, 'mean_word_length': 2.3519034354688952, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3592101', 'n_tokens_mistral': 1159, 'n_tokens_neox': 1126, 'n_words': 359}