<commit_before># -*- encoding : utf-8 -*-
class String
  # http://lucene.apache.org/core/old_versioned_docs/versions/2_9_1/queryparsersyntax.html
  #LUCENE_ESCAPE_REGEX = /(\+|-|&&|\|\||!|\(|\)|{|}|\[|\]|`|"|~|\?|:|\\|\/)/
  #LUCENE_ESCAPE_REGEX = /(\+|-|&&|\|\||!|\(|\)|{|}|\[|\]|`|"|~|\?|:|\\|\s)/

  KEYBOARDS = {
      en: 'qwertyuiop[]asdfghjkl;\'zxcvbnm,./',
      ru: 'йцукенгшщзхъфывапролджэячсмитьбю/'
  }

  unless defined? LUCENE_ESCAPE_REGEX
    LUCENE_ESCAPE_REGEX = /(\+|-|&&|\|\||!|\(|\)|{|}|\[|\]|\^|"|~|\*|\?|:|\\|\/)/

    def lucene_escape
      self.gsub(LUCENE_ESCAPE_REGEX, "\\\\\\1")
    end
  end

  def capitalize_first
    self.mb_chars[0].capitalize + self.mb_chars[1..-1]
  end

  def is_int?
    self =~ /^[-+]?[0-9]+$/
  end

  def is_number?
    true if Float(self) rescue false
  end

  def to_utc
    begin
      Time.zone.parse(self).utc
    rescue
      Time.now.utc
    end
  end

  def no_html
    str = self.dup
    str.gsub!(/<\/?[^>]*>/, '')
    str.strip!
    str.gsub!('&nbsp;', ' ')
    str
  end

  def tr_lang(from=nil, to=nil)
    return '' if self.blank?

    unless from || to
      if KEYBOARDS[:en].index(self[0])
        from, to = :en, :ru
      elsif KEYBOARDS[:ru].index(self[0])
        from, to = :ru, :en
      else
        from, to = :en, :ru
      end
    end

    self.tr(KEYBOARDS[from], KEYBOARDS[to])
  end

  def count_words
    clean_text.scan(/(\p{Alnum}+([-'.]\p{Alnum}+)*)/u).size
  end

  def words_count
		frequencies = Hash.new(0)  
		downcase.scan(/(\w+([-'.]\w+)*)/) { |word, ignore| frequencies[word] += 1 }
		frequencies
	end
	
	def self.randomize(length = 8)
	  Array.new(length) { (rand(122-97) + 97).chr }.join
  end

  def clean_text
    coder = HTMLEntities.new
    coder.decode(self.no_html)
  end

end

unless ''.respond_to?(:each)
  String.class_eval do
    def each &block
      self.lines &block
    end
  end
end<commit_msg>add mb_upcase and mb_downcase methods to string for utf8 upcase downcase<commit_after># -*- encoding : utf-8 -*-
class String
  # http://lucene.apache.org/core/old_versioned_docs/versions/2_9_1/queryparsersyntax.html
  #LUCENE_ESCAPE_REGEX = /(\+|-|&&|\|\||!|\(|\)|{|}|\[|\]|`|"|~|\?|:|\\|\/)/
  #LUCENE_ESCAPE_REGEX = /(\+|-|&&|\|\||!|\(|\)|{|}|\[|\]|`|"|~|\?|:|\\|\s)/

  KEYBOARDS = {
      en: 'qwertyuiop[]asdfghjkl;\'zxcvbnm,./',
      ru: 'йцукенгшщзхъфывапролджэячсмитьбю/'
  }

  unless defined? LUCENE_ESCAPE_REGEX
    LUCENE_ESCAPE_REGEX = /(\+|-|&&|\|\||!|\(|\)|{|}|\[|\]|\^|"|~|\*|\?|:|\\|\/)/

    def lucene_escape
      self.gsub(LUCENE_ESCAPE_REGEX, "\\\\\\1")
    end
  end

  def capitalize_first
    self.mb_chars[0].capitalize + self.mb_chars[1..-1]
  end

  def is_int?
    self =~ /^[-+]?[0-9]+$/
  end

  def is_number?
    true if Float(self) rescue false
  end

  def to_utc
    begin
      Time.zone.parse(self).utc
    rescue
      Time.now.utc
    end
  end

  def no_html
    str = self.dup
    str.gsub!(/<\/?[^>]*>/, '')
    str.strip!
    str.gsub!('&nbsp;', ' ')
    str
  end

  def tr_lang(from=nil, to=nil)
    return '' if self.blank?

    unless from || to
      if KEYBOARDS[:en].index(self[0])
        from, to = :en, :ru
      elsif KEYBOARDS[:ru].index(self[0])
        from, to = :ru, :en
      else
        from, to = :en, :ru
      end
    end

    self.tr(KEYBOARDS[from], KEYBOARDS[to])
  end

  def count_words
    clean_text.scan(/(\p{Alnum}+([-'.]\p{Alnum}+)*)/u).size
  end

  def words_count
		frequencies = Hash.new(0)  
		downcase.scan(/(\w+([-'.]\w+)*)/) { |word, ignore| frequencies[word] += 1 }
		frequencies
	end
	
	def self.randomize(length = 8)
	  Array.new(length) { (rand(122-97) + 97).chr }.join
  end

  def clean_text
    coder = HTMLEntities.new
    coder.decode(self.no_html)
  end

  def mb_upcase
    mb_chars.upcase.to_s
  end

  def mb_downcase
    mb_chars.downcase.to_s
  end
end

unless ''.respond_to?(:each)
  String.class_eval do
    def each &block
      self.lines &block
    end
  end
end
{'fraction_non_alphanumeric': 0.2161890397184515, 'fraction_numerical': 0.011312217194570135, 'mean_word_length': 2.4964850615114234, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3806835', 'n_tokens_mistral': 1827, 'n_tokens_neox': 1726, 'n_words': 281}