<commit_before># coding: utf-8

require 'sms_toolkit/rails/engine'

# Mixins
module SmsToolkit
  
  # Main toolkit class
  class PhoneNumbers

    # Normalizes the US phone number.
    # - Converts the number in the format "+1nnnnnnnnnn" or "001nnnnnnnnnn" to 10-digits.
    # - Leaves short-codes intact (e.g. "nnnnnn")
    def self.normalize(number)
      return number if number.blank?
      
      # Cleanup
      number = number.strip
      number = (number[0, 1] == '+' ? '+' : '') + number.gsub(/[^0-9]/, '')      
      
      number.gsub(/^\+?1?(\d{10})$/, '\1').gsub(/^001?(\d+)$/, '\1')
    end

    # Formats the 10-digit number or the short-code as the US number.
    def self.format(number)
      return number if number.blank?
      
      if /^\s*\+?[\d\-\s\(\)]+\s*$/ =~ number
        number = normalize(number)
        
        if /^\d{10}$/ =~ number
          format_10_digit_number(number)
        elsif /^\d{1,9}$/ =~ number
          format_short_code(number)
        else
          number
        end
      else
        number
      end
    end

    private

    # Formats 10-digit number
    def self.format_10_digit_number(number)
      "(#{number[0, 3]}) #{number[3, 3]}-#{number[6, 4]}"
    end

    # Formats short-code
    def self.format_short_code(number)
      number
    end

  end

  class Message
    attr_accessor :text

    def initialize(text)
      self.text = text
    end

    def to_ascii
      ascii_text = self.text.dup

      ascii_text.gsub!(/[’′ʻ´`‘’]/, "'")
      ascii_text.gsub!(/[“”‹›«»]/, '"')
      ascii_text.gsub!(/[‐−‒–—―]/, '-')
      ascii_text.gsub!(/[…⋯]/, "...")
      ascii_text.gsub!(/[◦‣∙◘◙]/, '*')
      ascii_text.gsub!(/[º°]/, 'o')
      ascii_text.gsub!(/[\u3133\u200b\u00a0\u2028\u{1f44d}\u{1f383}\u{1f44f}\u{1f483}]/, '')
      ascii_text.gsub!(/[\u{1f600}-\u{1f64f}]/, '') # smiles filtering out
      ascii_text.gsub!(/\u189/, '1/2')
      ascii_text.gsub!(/\u188/, '1/4')
      ascii_text.gsub!(/\u190/, '3/4')
      ascii_text.gsub!(/\u8531/, '1/3')
      ascii_text.gsub!(/\u8532/, '2/3')
      ascii_text.tr!("ÀÁÂÃÄÅàáâãäåĀāĂăĄąÇçĆćĈĉĊċČčÐðĎďĐđÈÉÊËèéêëĒēĔĕĖėĘęĚěĜĝĞğĠġĢģĤĥĦħÌÍÎÏìíîïĨĩĪīĬĭĮįİıĴĵĶķĸĹĺĻļĽľĿŀŁłÑñŃńŅņŇňŉŊŋÒÓÔÕÖØòóôõöøŌōŎŏŐőŔŕŖŗŘřŚśŜŝŞşŠšſŢţŤťŦŧÙÚÛÜùúûüŨũŪūŬŭŮůŰűŲųŴŵÝýÿŶŷŸŹźŻżŽž",
                     "AAAAAAaaaaaaAaAaAaCcCcCcCcCcDdDdDdEEEEeeeeEeEeEeEeEeGgGgGgGgHhHhIIIIiiiiIiIiIiIiIiJjKkkLlLlLlLlLlNnNnNnNnnNnOOOOOOooooooOoOoOoRrRrRrSsSsSsSssTtTtTtUUUUuuuuUuUuUuUuUuUuWwYyyYyYZzZzZz")
      ascii_text.gsub!(/\&/, '')
      ascii_text.gsub!(/\P{ASCII}/, '')

      ascii_text
    end

    def to_ascii_with_diff
      ascii = to_ascii

      diff = self.text.dup.split('') - ascii.split('')
<commit_msg>fractions processing was added. small typo fix<commit_after># coding: utf-8

require 'sms_toolkit/rails/engine'

# Mixins
module SmsToolkit
  
  # Main toolkit class
  class PhoneNumbers

    # Normalizes the US phone number.
    # - Converts the number in the format "+1nnnnnnnnnn" or "001nnnnnnnnnn" to 10-digits.
    # - Leaves short-codes intact (e.g. "nnnnnn")
    def self.normalize(number)
      return number if number.blank?
      
      # Cleanup
      number = number.strip
      number = (number[0, 1] == '+' ? '+' : '') + number.gsub(/[^0-9]/, '')      
      
      number.gsub(/^\+?1?(\d{10})$/, '\1').gsub(/^001?(\d+)$/, '\1')
    end

    # Formats the 10-digit number or the short-code as the US number.
    def self.format(number)
      return number if number.blank?
      
      if /^\s*\+?[\d\-\s\(\)]+\s*$/ =~ number
        number = normalize(number)
        
        if /^\d{10}$/ =~ number
          format_10_digit_number(number)
        elsif /^\d{1,9}$/ =~ number
          format_short_code(number)
        else
          number
        end
      else
        number
      end
    end

    private

    # Formats 10-digit number
    def self.format_10_digit_number(number)
      "(#{number[0, 3]}) #{number[3, 3]}-#{number[6, 4]}"
    end

    # Formats short-code
    def self.format_short_code(number)
      number
    end

  end

  class Message
    attr_accessor :text

    def initialize(text)
      self.text = text
    end

    def to_ascii
      ascii_text = self.text.dup

      ascii_text.gsub!(/[’′ʻ´`‘’]/, "'")
      ascii_text.gsub!(/[“”‹›«»]/, '"')
      ascii_text.gsub!(/[‐−‒–—―]/, '-')
      ascii_text.gsub!(/[…⋯]/, "...")
      ascii_text.gsub!(/[◦‣∙◘◙]/, '*')
      ascii_text.gsub!(/[º°]/, 'o')
      ascii_text.gsub!(/[\u3133\u200b\u00a0\u2028\u{1f44d}\u{1f383}\u{1f44f}\u{1f483}]/, '')
      ascii_text.gsub!(/[\u{1f600}-\u{1f64f}]/, '') # smiles filtering out
      ascii_text.gsub!(/\u{189}/, '1/2')
      ascii_text.gsub!(/\u{188}/, '1/4')
      ascii_text.gsub!(/\u{190}/, '3/4')
      ascii_text.gsub!(/\u8531/, '1/3')
      ascii_text.gsub!(/\u8532/, '2/3')
      ascii_text.tr!("ÀÁÂÃÄÅàáâãäåĀāĂăĄąÇçĆćĈĉĊċČčÐðĎďĐđÈÉÊËèéêëĒēĔĕĖėĘęĚěĜĝĞğĠġĢģĤĥĦħÌÍÎÏìíîïĨĩĪīĬĭĮįİıĴĵĶķĸĹĺĻļĽľĿŀŁłÑñŃńŅņŇňŉŊŋÒÓÔÕÖØòóôõöøŌōŎŏŐőŔŕŖŗŘřŚśŜŝŞşŠšſŢţŤťŦŧÙÚÛÜùúûüŨũŪūŬŭŮůŰűŲųŴŵÝýÿŶŷŸŹźŻżŽž",
                     "AAAAAAaaaaaaAaAaAaCcCcCcCcCcDdDdDdEEEEeeeeEeEeEeEeEeGgGgGgGgHhHhIIIIiiiiIiIiIiIiIiJjKkkLlLlLlLlLlNnNnNnNnnNnOOOOOOooooooOoOoOoRrRrRrSsSsSsSssTtTtTtUUUUuuuuUuUuUuUuUuUuWwYyyYyYZzZzZz")
      ascii_text.gsub!(/\&/, '')
      ascii_text.gsub!(/\P{ASCII}/, '')

      ascii_text
    end

    def to_ascii_with_diff
      ascii = to_ascii

      diff = self.text.dup.split('') - ascii.split('')

{'fraction_non_alphanumeric': 0.16847726017308046, 'fraction_numerical': 0.036457374332535444, 'mean_word_length': 2.513583441138422, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2836404', 'n_tokens_mistral': 2797, 'n_tokens_neox': 2677, 'n_words': 320}