<commit_before># encoding: utf-8

require 'phonetic/algorithm'

module Phonetic
  # The Double Metaphone phonetic encoding algorithm is the second generation
  # of the Metaphone algorithm. Its original implementation was described
  # by <NAME> in the June 2000 issue of C/C++ Users Journal.
  #
  # This implementation based on the PHP implementation by <NAME>
  # and contains modifications of algorithm by <NAME>.
  # @see http://swoodbridge.com/DoubleMetaPhone/
  #      PHP implementation by <NAME>
  # @see http://aspell.net/metaphone/dmetaph.cpp
  #      C++ implementation with modifications by <NAME>
  # @example
  #    Phonetic::DoubleMetaphone.encode('czerny') # => ['SRN', 'XRN']
  #    Phonetic::DoubleMetaphone.encode('dumb')   # => ['TM', 'TM']
  #    Phonetic::DoubleMetaphone.encode('edgar')  # => ['ATKR', 'ATKR']
  #    # or use alias:
  #    Phonetic::Metaphone2.encode('czerny') # => ['SRN', 'XRN']
  #    Phonetic::Metaphone2.encode('dumb')   # => ['TM', 'TM']
  #    Phonetic::Metaphone2.encode('edgar')  # => ['ATKR', 'ATKR']
  class DoubleMetaphone < Algorithm
    # Encode word to its Double Metaphone code.
    def self.encode_word(word, options = { size: 4 })
      code_size = options[:size] || 4
      w = word.strip.upcase
      code = ['', '']
      def code.add(primary, secondary)
        self[0] += primary
        self[1] += secondary
      end
      i = 0
      len = w.size
      last = len - 1
      # pad the original string so that we can index beyond the edge of the world
      w += ' ' * 5
      # skip these when at start of word
      i += 1 if w[0, 2] =~ /[GKP]N|WR|PS/
      # initial 'X' is pronounced 'Z' e.g. 'Xavier'
      if w[0] == 'X'
        code.add 'S', 'S'
        i += 1
      end
      while i < len && (code.first.size < code_size || code.last.size < code_size)
        case w[i]
        when 'A', 'E', 'I', 'O', 'U', 'Y'
          code.add 'A', 'A' if i == 0 # all init vowels now map to 'A'
          i += 1
        when 'B'
          # "-mb", e.g", "dumb", already skipped over...
          i += gen_encode(w, i, 'P', 'P', code)
        when 'Ç', 'ç'
          code.add 'S', 'S'
          i += 1
        when 'C'
          i += encode_c(w, i, len, code)
        when 'D'
          i += encode_d(w, i, len, code)
        when 'F', 'K', 'N'
          i += gen_encode(w, i, w[i], w[i], code)
        when 'G'
          i += encode_g(w, i, len, code)
        when 'H'
          i += encode_h(w, i, len, code)
        when 'J'
          i += encode_j(w, i, len, code)
        when 'L'
          i += encode_l(w, i, len, code)
        when 'M'
          i += encode_m(w, i, len, code)
        when 'Ñ', 'ñ'
          code.add 'N', 'N'
          i += 1
        when 'P'
          i += encode_p(w, i, len, code)
        when 'Q'
          i += gen_encode(w, i, 'K', 'K', code)
        when 'R'
          i += encode_r(w, i, len, code)
        when 'S'
          i += encode_s(w, i, len, code)
        when 'T'
          i += encode_t(w, i, len, code)
        when 'V'
          i += gen_encode(w, i, 'F', 'F', code)
        when 'W'
          i += encode_w(w, i, len, code)
        when 'X'
          i += encode_x(w, i, len, code)
        when 'Z'
          i += encode_z(w, i, len, code)
        else
          i += 1
        end
      end
      [code.first[0, code_size], code.last[0, code_size]]
    end

    def self.encode(str, options = { size: 4 })
      encode_word(str, options)
    end

    private

    def self.gen_encode(w, i, primary, secondary, code)
      code.add primary, secondary
      w[i + 1] == w[i] ? 2 : 1
    end

    def self.encode_c(w, i, len, code)
      r = 0
      case
      # various germanic
      when c_germanic?(w, i)
        code.add 'K', 'K'
        r += 2
      # special case 'caesar'
      when i == 0 && w[i, 6] == 'CAESAR'
        code.add 'S', 'S'
        r += 2
      when w[i, 2] == 'CH'
        encode_ch(w, i, len, code)
        r += 2
      when w[i, 2] == 'CZ' && !(i > 1 && w[i - 2, 4] == 'WICZ')
        # e.g, 'czerny'
        code.add 'S', 'X'
        r += 2
      when w[i + 1, 3] == 'CIA'
        # e.g., 'focaccia'
        code.add 'X', 'X'
        r += 3
      # double 'C', but not if e.g. 'McClellan'
      when w[i, 2] == 'CC' && !(i == 1 && w[0] == 'M')
        r += encode_cc(w, i, code) + 2
      when w[i, 2] =~ /C[KGQ]/
        code.add 'K', 'K'
        r += 2
      when w[i, 2] =~ /C[IEY]/
        # italian vs. english
        if w[i, 3] =~ /CI[OEA]/
          code.add 'S', 'X'
        else
          code.add 'S', 'S'
        end
        r += 2
      else
        code.add 'K', 'K'
        # name sent in '<NAME>', '<NAME>'
        if w[i + 1, 2] =~ /\s[CQG]/
          r += 3
        elsif w[i + 1] =~ /[CKQ]/ && w[i + 1, 2] !~ /C[EI]/
          r += 2
        else
          r += 1
        end
      end
      r
    end<commit_msg>refactor double metaphone class 6<commit_after># encoding: utf-8

require 'phonetic/algorithm'

module Phonetic
  # The Double Metaphone phonetic encoding algorithm is the second generation
  # of the Metaphone algorithm. Its original implementation was described
  # by <NAME> in the June 2000 issue of C/C++ Users Journal.
  #
  # This implementation based on the PHP implementation by <NAME>
  # and contains modifications of algorithm by <NAME>.
  # @see http://swoodbridge.com/DoubleMetaPhone/
  #      PHP implementation by <NAME>
  # @see http://aspell.net/metaphone/dmetaph.cpp
  #      C++ implementation with modifications by <NAME>
  # @example
  #    Phonetic::DoubleMetaphone.encode('czerny') # => ['SRN', 'XRN']
  #    Phonetic::DoubleMetaphone.encode('dumb')   # => ['TM', 'TM']
  #    Phonetic::DoubleMetaphone.encode('edgar')  # => ['ATKR', 'ATKR']
  #    # or use alias:
  #    Phonetic::Metaphone2.encode('czerny') # => ['SRN', 'XRN']
  #    Phonetic::Metaphone2.encode('dumb')   # => ['TM', 'TM']
  #    Phonetic::Metaphone2.encode('edgar')  # => ['ATKR', 'ATKR']
  class DoubleMetaphone < Algorithm
    # Encode word to its Double Metaphone code.
    def self.encode_word(word, options = { size: 4 })
      code_size = options[:size] || 4
      w = word.strip.upcase
      code = ['', '']
      def code.add(primary, secondary)
        self[0] += primary
        self[1] += secondary
      end
      i = 0
      len = w.size
      last = len - 1
      # pad the original string so that we can index beyond the edge of the world
      w += ' ' * 5
      i += encode_start_of_word(w, code)
      while i < len && (code.first.size < code_size || code.last.size < code_size)
        case w[i]
        when 'A', 'E', 'I', 'O', 'U', 'Y'
          i += 1
        when 'B'
          # "-mb", e.g", "dumb", already skipped over...
          i += gen_encode(w, i, 'P', 'P', code)
        when 'Ç', 'ç'
          code.add 'S', 'S'
          i += 1
        when 'C'
          i += encode_c(w, i, len, code)
        when 'D'
          i += encode_d(w, i, len, code)
        when 'F', 'K', 'N'
          i += gen_encode(w, i, w[i], w[i], code)
        when 'G'
          i += encode_g(w, i, len, code)
        when 'H'
          i += encode_h(w, i, len, code)
        when 'J'
          i += encode_j(w, i, len, code)
        when 'L'
          i += encode_l(w, i, len, code)
        when 'M'
          i += encode_m(w, i, len, code)
        when 'Ñ', 'ñ'
          code.add 'N', 'N'
          i += 1
        when 'P'
          i += encode_p(w, i, len, code)
        when 'Q'
          i += gen_encode(w, i, 'K', 'K', code)
        when 'R'
          i += encode_r(w, i, len, code)
        when 'S'
          i += encode_s(w, i, len, code)
        when 'T'
          i += encode_t(w, i, len, code)
        when 'V'
          i += gen_encode(w, i, 'F', 'F', code)
        when 'W'
          i += encode_w(w, i, len, code)
        when 'X'
          i += encode_x(w, i, len, code)
        when 'Z'
          i += encode_z(w, i, len, code)
        else
          i += 1
        end
      end
      [code.first[0, code_size], code.last[0, code_size]]
    end

    def self.encode(str, options = { size: 4 })
      encode_word(str, options)
    end

    private

    def self.encode_start_of_word(w, code)
      i = 0
      # skip these when at start of word
      if w[0, 2] =~ /[GKP]N|WR|PS/
        i = 1
      # initial 'X' is pronounced 'Z' e.g. 'Xavier'
      elsif w[0] == 'X'
        code.add 'S', 'S'
        i = 1
      elsif w[0] =~ /[AEIOUY]/
        code.add 'A', 'A' # all init vowels now map to 'A'
        i = 1
      elsif w[0, 6] == 'CAESAR' # special case 'caesar'
        code.add 'S', 'S'
        i = 1
      end
      i
    end

    def self.gen_encode(w, i, primary, secondary, code)
      code.add primary, secondary
      w[i + 1] == w[i] ? 2 : 1
    end

    def self.encode_c(w, i, len, code)
      r = 1
      case
      # various germanic
      when c_germanic?(w, i)
        code.add 'K', 'K'
        r += 1
      when w[i, 2] == 'CH'
        encode_ch(w, i, len, code)
        r += 1
      when w[i, 2] == 'CZ' && !(i > 1 && w[i - 2, 4] == 'WICZ')
        # e.g, 'czerny'
        code.add 'S', 'X'
        r += 1
      when w[i + 1, 3] == 'CIA'
        # e.g., 'focaccia'
        code.add 'X', 'X'
        r += 2
      # double 'C', but not if e.g. 'McClellan'
      when w[i, 2] == 'CC' && !(i == 1 && w[0] == 'M')
        r += encode_cc(w, i, code) + 1
      when w[i, 2] =~ /C[KGQ]/
        code.add 'K', 'K'
        r += 1
      when w[i, 2] =~ /C[IEY]/
        # italian vs. english
        if w[i, 3] =~ /CI[OEA]/
          code.add 'S', 'X'
        else
          code.add 'S', 'S'
        end
        r += 1
      else
        code.add 'K', 'K'
        # name sent in '<NAME>', '<NAME>'
        if w[i + 1, 2] =~ /\s[CQG]/
          r += 2
        elsif w[i + 1] =~ /[CKQ]/ && w[i + 1, 2] !~ /C[EI]/
          r += 1
        end
      end
      r
    end
{'fraction_non_alphanumeric': 0.1679342865835108, 'fraction_numerical': 0.012980427948483926, 'mean_word_length': 1.635489043292357, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1057411', 'n_tokens_mistral': 3809, 'n_tokens_neox': 3677, 'n_words': 1098}