<commit_before>#encoding: utf-8
require 'json'
require "japanese_deinflector/version"

class JapaneseDeinflector
  def initialize
    File.open(File.join(File.expand_path(File.dirname(__FILE__)), 'data/deinflect.json')) do |f|
      rules_and_reasons = JSON.parse(f.read, :symbolize_names => true)
      @reasons = rules_and_reasons[:reasons]
      # Convert hash keys to integers
      @rules = {}
      # Convert hash keys from something like :"9" -> 9
      rules_and_reasons[:rules].each do |size, rules|
        @rules[size.to_s.to_i] = rules
      end
    end
  end

  def deinflect(word)
    possibilities = []
    rules_less_than_size(word.size).each do |size, rules|
      ending = word[-size..-1]
      rules.each do |rule|
        next  unless ending == rule[:from]
        deinflected_word = "#{word[0..-size-1]}#{rule[:to]}"
        next  if possibilities.include?(deinflected_word)
        # Weight is between 0 and 1, 1 being a higher chance of actual deinflection
        weight = (Float(size) / word.size).round(3)
        reason = @reasons[rule[:reason_id]]
        possibilities << {weight: weight, word: deinflected_word, reason: reason}
      end
    end
    possibilities
  end

  private

  def rules_less_than_size(max_size)<commit_msg>Support older Ruby versions<commit_after>#encoding: utf-8
require 'json'
require "japanese_deinflector/version"

class JapaneseDeinflector
  def initialize
    File.open(File.join(File.expand_path(File.dirname(__FILE__)), 'data/deinflect.json')) do |f|
      rules_and_reasons = JSON.parse(f.read, :symbolize_names => true)
      @reasons = rules_and_reasons[:reasons]
      # Convert hash keys to integers
      @rules = {}
      # Convert hash keys from something like :"9" -> 9
      rules_and_reasons[:rules].each do |size, rules|
        @rules[size.to_s.to_i] = rules
      end
    end
  end

  def deinflect(word)
    possibilities = []
    rules_less_than_size(word.size).each do |size, rules|
      ending = word[-size..-1]
      rules.each do |rule|
        next  unless ending == rule[:from]
        deinflected_word = "#{word[0..-size-1]}#{rule[:to]}"
        next  if possibilities.include?(deinflected_word)
        # Weight is between 0 and 1, 1 being a higher chance of actual deinflection
        weight = (Float(size) / word.size).round(3)
        reason = @reasons[rule[:reason_id]]
        possibilities << {:weight => weight, :word => deinflected_word, :reason => reason}
      end
    end
    possibilities
  end

  private

  def rules_less_than_size(max_size)
{'fraction_non_alphanumeric': 0.11797090051120723, 'fraction_numerical': 0.007864726700747149, 'mean_word_length': 3.0574162679425836, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4709976', 'n_tokens_mistral': 827, 'n_tokens_neox': 822, 'n_words': 196}