<commit_before># Author::    <NAME>  (mailto:<EMAIL>)
# Copyright:: Copyright (c) 2005 <NAME>
# License::   LGPL

require "set"

# These are extensions to the String class to provide convenience 
# methods for the Classifier package.
class String
  
  # Removes common punctuation symbols, returning a new string. 
  # E.g.,
  #   "Hello (greeting's), with {braces} < >...?".without_punctuation
  #   => "Hello  greetings   with  braces         "
  def without_punctuation
    tr( ',?.!;:"@#$%^&*()_=+[]{}\|<>/`~', " " ) .tr( "'\-", "")
  end
  
  # Return a Hash of strings => ints. Each word in the string is stemmed,
  # interned, and indexes to its frequency in the document.  
	def word_hash
		word_hash = clean_word_hash()
		symbol_hash = word_hash_for_symbols(gsub(/[\w]/," ").split)
		return word_hash.merge(symbol_hash)
	end

	# Return a word hash without extra punctuation or short symbols, just stemmed words
	def clean_word_hash
		word_hash_for_words gsub(/[^\w\s]/,"").split
	end
	
	private
	
	def word_hash_for_words(words)
		d = Hash.new(0)
		words.each do |word|
			word.downcase!
			key = word.stem.intern
			if ! CORPUS_SKIP_WORDS.include?(word) && word.length > 2
				d[key] += 1
			end
		end
		return d
	end


	def word_hash_for_symbols(words)
		d = Hash.new(0)
		words.each do |word|
			key = word.intern
			d[key] += 1
		end
		return d
	end
	
	CORPUS_SKIP_WORDS = Set.new([
      "a",
      "again",
      "all",
      "along",
      "are",
      "also",
      "an",
      "and",
      "as",
      "at",
      "but",
      "by",
      "came",
      "can",
      "cant",
      "couldnt",
      "did",
      "didn",
      "didnt",
      "do",
      "doesnt",
      "dont",
      "ever",
      "first",
      "from",<commit_msg>only stem words we aren't skipping<commit_after># Author::    <NAME>  (mailto:<EMAIL>)
# Copyright:: Copyright (c) 2005 <NAME>
# License::   LGPL

require "set"

# These are extensions to the String class to provide convenience 
# methods for the Classifier package.
class String
  
  # Removes common punctuation symbols, returning a new string. 
  # E.g.,
  #   "Hello (greeting's), with {braces} < >...?".without_punctuation
  #   => "Hello  greetings   with  braces         "
  def without_punctuation
    tr( ',?.!;:"@#$%^&*()_=+[]{}\|<>/`~', " " ) .tr( "'\-", "")
  end
  
  # Return a Hash of strings => ints. Each word in the string is stemmed,
  # interned, and indexes to its frequency in the document.  
	def word_hash
		word_hash = clean_word_hash()
		symbol_hash = word_hash_for_symbols(gsub(/[\w]/," ").split)
		return word_hash.merge(symbol_hash)
	end

	# Return a word hash without extra punctuation or short symbols, just stemmed words
	def clean_word_hash
		word_hash_for_words gsub(/[^\w\s]/,"").split
	end
	
	private
	
	def word_hash_for_words(words)
		d = Hash.new(0)
		words.each do |word|
			word.downcase!
			if ! CORPUS_SKIP_WORDS.include?(word) && word.length > 2
				d[word.stem.intern] += 1
			end
		end
		return d
	end


	def word_hash_for_symbols(words)
		d = Hash.new(0)
		words.each do |word|
			d[word.intern] += 1
		end
		return d
	end
	
	CORPUS_SKIP_WORDS = Set.new([
      "a",
      "again",
      "all",
      "along",
      "are",
      "also",
      "an",
      "and",
      "as",
      "at",
      "but",
      "by",
      "came",
      "can",
      "cant",
      "couldnt",
      "did",
      "didn",
      "didnt",
      "do",
      "doesnt",
      "dont",
      "ever",
      "first",
      "from",
{'fraction_non_alphanumeric': 0.1527658354829464, 'fraction_numerical': 0.005159071367153913, 'mean_word_length': 2.469184890656064, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3417595', 'n_tokens_mistral': 1385, 'n_tokens_neox': 1284, 'n_words': 334}