<commit_before>require 'yaml'

class Hash
  def deep_merge(other)
    # deep_merge by <NAME>, see http://www.ruby-forum.com/topic/142809
    merger = proc { |key, v1, v2| (Hash === v1 && Hash === v2) ? v1.merge(v2, &merger) : v2 }
    merge(other, &merger)
  end

  def set(keys, value)
    key = keys.shift
    if keys.empty?
      self[key] = value
    else
      self[key] ||= {}
      self[key].set keys, value
    end
  end

  if ENV['SORT']
    # copy of ruby's to_yaml method, prepending sort.
    # before each so we get an ordered yaml file
    def to_yaml( opts = {} )
      YAML::quick_emit( self, opts ) do |out|
        out.map( taguri, to_yaml_style ) do |map|
          sort.each do |k, v| #<- Adding sort.
            map.add( k, v )
          end
        end
      end
    end
  end
end

namespace :translate do

  def find_missing_translations(locale)
    keys        = []
    result      = []
    locale_hash = {}

    puts "Searching for missing translations for the locale: #{locale}"
    default_locale_files = Dir.glob(Rails.root.join('vendor', 'plugins', '*', 'config', 'locales', "#{locale}.yml"))
    default_locale_files += Dir.glob(File.join(Translate.locales_dir, "**","#{locale}.yml"))

    default_locale_files.each do |locale_file_name|
      yaml = YAML::load(File.open(locale_file_name))
      locale_hash = locale_hash.deep_merge(yaml[locale.to_s])
    end

    lookup_pattern = /\b(?:I18n\.t|I18n\.translate|t)(?:\s|\():?'([a-z0-9_]*.[a-z0-9_.]+)'\)?/
    (Dir.glob(File.join("app", "**","*.{erb,rb}")) + Dir.glob(File.join("vendor", "plugins", "**", "app", "**", "*.{erb,rb}"))).each do |file_name|
      File.open(file_name, "r+").each do |line|
        line.scan(lookup_pattern) do |key_string|
          # qualify the namespace if beginning with . like t('.log_out')
          if key_string.first =~ /^\./
            namespace = file_name.gsub(/vendor\/plugins\/.+?\//, "").gsub(/^app\/(models|views|controllers|helpers)\//, '').split('/')
            namespace = namespace | [namespace.pop.gsub(/^\_/, '').split('.').first]
            key_string = ["#{namespace.join('.')}#{key_string.first}"]
          end
          
          unless key_exist?(key_string.first.split("."), locale_hash)
            result << "#{key_string} in \t  #{file_name} \t is not in any #{locale} locale file"
          end
        end
      end unless file_name =~ /translate\/spec/
    end
    puts result.empty? ? "No missing translations for locale: #{locale}" : "#{result.join("\n")}\n\nNumber of missing translations for #{locale}: #{result.length}"
  end

  desc "Show I18n keys that are missing in the specified locale YAML file. Defaults to I18n.default_locale, unless LOCALE env is specified"
  task :lost_in_translation => :environment do
    locale      = ENV['LOCALE'] || I18n.default_locale
    find_missing_translations(locale)
  end

  desc "Show I18n keys that are missing in all locale YAML files."<commit_msg>Fixed translate task to find locales in new locations.<commit_after>require 'yaml'

class Hash
  def deep_merge(other)
    # deep_merge by <NAME>, see http://www.ruby-forum.com/topic/142809
    merger = proc { |key, v1, v2| (Hash === v1 && Hash === v2) ? v1.merge(v2, &merger) : v2 }
    merge(other, &merger)
  end

  def set(keys, value)
    key = keys.shift
    if keys.empty?
      self[key] = value
    else
      self[key] ||= {}
      self[key].set keys, value
    end
  end

  if ENV['SORT']
    # copy of ruby's to_yaml method, prepending sort.
    # before each so we get an ordered yaml file
    def to_yaml( opts = {} )
      YAML::quick_emit( self, opts ) do |out|
        out.map( taguri, to_yaml_style ) do |map|
          sort.each do |k, v| #<- Adding sort.
            map.add( k, v )
          end
        end
      end
    end
  end
end

namespace :translate do

  def find_missing_translations(locale)
    keys        = []
    result      = []
    locale_hash = {}

    puts "Searching for missing translations for the locale: #{locale}"
    default_locale_files = Dir.glob(Rails.root.join('vendor', 'plugins', '*', 'config', 'locales', "#{locale}.yml").to_s)
    default_locale_files += Dir.glob(Refinery.root.join('vendor', 'plugins', '*', 'config', 'locales', "#{locale}.yml").to_s)
    default_locale_files += Dir.glob(Refinery.root.join('vendor', 'refinerycms', '*', 'config', 'locales', "#{locale}.yml").to_s)
    default_locale_files += Dir.glob(File.join(Translate.locales_dir, "**","#{locale}.yml").to_s)

    default_locale_files.uniq.each do |locale_file_name|
      yaml = YAML::load(File.open(locale_file_name))
      locale_hash = locale_hash.deep_merge(yaml[locale.to_s])
    end

    lookup_pattern = /\b(?:I18n\.t|I18n\.translate|t)(?:\s|\():?'([a-z0-9_]*.[a-z0-9_.]+)'\)?/
    templates = Dir.glob(Rails.root.join('app', '**', '*.{erb,rb}').to_s)
    templates += Dir.glob(Rails.root.join.join('vendor', 'plugins', '**', 'app', '**', '*.{erb,rb}').to_s)
    templates += Dir.glob(Refinery.root.join('vendor', 'plugins', '**', 'app', '**', '*.{erb,rb}').to_s)
    templates += Dir.glob(Refinery.root.join('vendor', 'refinerycms', '**', 'app', '**', '*.{erb,rb}').to_s)

    templates.reject{|t| t =~ /\/lib\/generators\/.+?\/templates\//}.uniq.each do |file_name|
      File.open(file_name, "r+").each do |line|
        line.scan(lookup_pattern) do |key_string|
          # qualify the namespace if beginning with . like t('.log_out')
          if key_string.first =~ /^\./
            namespace = file_name.split("app/").last.split(/^.+?\//).last.split('/')
            namespace = namespace | [namespace.pop.gsub(/^\_/, '').split('.').first]
            key_string = ["#{namespace.join('.')}#{key_string.first}"]
          end

          unless key_exist?(key_string.first.split("."), locale_hash)
            result << "#{key_string} in \t  #{file_name} \t is not in any #{locale} locale file"
          end
        end
      end unless file_name =~ /translate\/spec/
    end
    puts result.empty? ? "No missing translations for locale: #{locale}" : "#{result.join("\n\n")}\n\nNumber of missing translations for #{locale}: #{result.length}"
  end

  desc "Show I18n keys that are missing in the specified locale YAML file. Defaults to I18n.default_locale, unless LOCALE env is specified"
  task :lost_in_translation => :environment do
    locale      = ENV['LOCALE'] || I18n.default_locale
    find_missing_translations(locale)
  end

  desc "Show I18n keys that are missing in all locale YAML files."
{'fraction_non_alphanumeric': 0.17550077041602466, 'fraction_numerical': 0.008936825885978428, 'mean_word_length': 2.9008413461538463, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3742868', 'n_tokens_mistral': 2282, 'n_tokens_neox': 2203, 'n_words': 514}