<commit_before>namespace :i18n do
  module HashExtensions
    def flatten_keys(result={}, prefix='')
      each_pair do |k, v|
        if v.is_a?(Hash)
          v.flatten_keys(result, "#{prefix}#{k}.")
        else
          result["#{prefix}#{k}"] = v
        end
      end
      result
    end

    def expand_keys(result = {})
      each_pair do |k, v|
        parts = k.split('.')
        last = parts.pop
        parts.inject(result){ |h, k2| h[k2] ||= {}}[last] = v
      end
      result
    end

    def to_ordered
      keys.sort_by(&:to_s).inject ActiveSupport::OrderedHash.new do |h, k|
        v = fetch(k)
        h[k] = v.is_a?(Hash) ? v.to_ordered : v
        h
      end
    end
  end

  def infer_scope(filename)
    case filename
      when /app\/views\/.*\.handlebars\z/
        filename.gsub(/.*app\/views\/jst\/_?|\.handlebars\z/, '').underscore.gsub(/\/_?/, '.')
      when /app\/controllers\//
        scope = filename.gsub(/.*app\/controllers\/|controller.rb/, '').gsub(/\/_?|_\z/, '.')
        scope == 'application.' ? '' : scope
      when /app\/messages\//
        filename.gsub(/.*app\/|erb/, '').gsub(/\/_?/, '.')
      when /app\/models\//
        scope = filename.gsub(/.*app\/models\/|rb/, '')
        STI_SUPERCLASSES.include?(scope) ? '' : scope
      when /app\/views\//
        filename.gsub(/.*app\/views\/|(html\.|fbml\.)?erb\z/, '').gsub(/\/_?/, '.')
      else
        ''
    end
  end

  desc "Verifies all translation calls"
  task :check => :environment do
    Bundler.require :i18n_tools
    only = if ENV['ONLY']
      ENV['ONLY'].split(',').map{ |path|
        path = '**/' + path if path =~ /\*/
        path = './' + path unless path =~ /\A.?\//
        if path =~ /\*/
          path = Dir.glob(path)
        elsif path !~ /\.(e?rb|js)\z/
          path = Dir.glob(path + '/**/*')
        end
        path
      }.flatten
    end

    STI_SUPERCLASSES = (`grep '^class.*<' ./app/models/*rb|grep -v '::'|sed 's~.*< ~~'|sort|uniq`.split("\n") - ['OpenStruct', 'Tableless']).
      map{ |name| name.underscore + '.' }

    COLOR_ENABLED = ($stdout.tty? rescue false)
    def color(text, color_code)
      COLOR_ENABLED ? "#{color_code}#{text}\e[0m" : text
    end

    def green(text)
      color(text, "\e[32m")
    end

    def red(text)
      color(text, "\e[31m")
    end

    @errors = []
    def process_files(files)
      files.each do |file|
        begin
          print green "." if yield file
        rescue SyntaxError, StandardError
          @errors << "#{$!}\n#{file}"
          print red "F"
        end
      end
    end

    t = Time.now

    I18n.available_locales
    stringifier = proc { |hash, (key, value)|
      hash[key.to_s] = value.is_a?(Hash) ?
        value.inject({}, &stringifier) :
        value
      hash
    }
    @translations = I18n.backend.send(:translations)[:en].inject({}, &stringifier)


    # Ruby
    files = (Dir.glob('./*') - ['./vendor'] + ['./vendor/plugins'] - ['./guard', './tmp']).map { |d| Dir.glob("#{d}/**/*rb") }.flatten.
      reject{ |file| file =~ %r{\A\./(rb-fsevent|vendor/plugins/rails_xss|vendor/plugins/cells|db|spec)/} }
    files &= only if only
    file_count = files.size
    rb_extractor = I18nExtraction::RubyExtractor.new(:translations => @translations)
    process_files(files) do |file|
      source = File.read(file)
      source = Erubis::Eruby.new(source).src if file =~ /\.erb\z/

      sexps = RubyParser.new.parse(source)
      rb_extractor.scope = infer_scope(file)
      rb_extractor.in_html_view = (file =~ /\.(html|facebook)\.erb\z/)
      rb_extractor.process(sexps)
    end


    # JavaScript
    files = (Dir.glob('./public/javascripts/**/*.js') + Dir.glob('./app/views/**/*.erb')).
      reject{ |file| file =~ /\A\.\/public\/javascripts\/(i18nObj.js|i18n.js|jst\/|translations\/|compiled\/handlebars_helpers.js|tinymce\/jscripts\/tiny_mce(.*\/langs|\/tiny_mce\w*\.js))/ }
    files &= only if only
    js_extractor = I18nExtraction::JsExtractor.new(:translations => @translations)
    process_files(files) do |file|
      t2 = Time.now
      ret = js_extractor.process(File.read(file), :erb => (file =~ /\.erb\z/), :filename => file)
      file_count += 1 if ret
      puts "#{file} #{Time.now - t2}" if Time.now - t2 > 1
      ret
    end


    # Handlebars
    files = Dir.glob('./app/views/jst/**/*.handlebars')
    files &= only if only
    handlebars_extractor = I18nExtraction::HandlebarsExtractor.new(:translations => @translations)
    process_files(files) do |file|
      file_count += 1 if handlebars_extractor.process(File.read(file), infer_scope(file))
    end

    print "\n\n"
    failure = @errors.size > 0

    @errors.each_index do |i|
      puts "#{i+1})"
      puts red @errors[i]
      print "\n"
    end

    print "Finished in #{Time.now - t} seconds\n\n"
    total_strings = rb_extractor.total_unique + js_extractor.total_unique + handlebars_extractor.total_unique
    puts send((failure ? :red : :green), "#{file_count} files, #{total_strings} strings, #{@errors.size} failures")
    raise "check command encountered errors" if failure
  end

  desc "Generates a new en.yml file for all translations"
  task :generate => :check do
    yaml_dir = './config/locales/generated'
    FileUtils.mkdir_p(File.join(yaml_dir))
    yaml_file = File.join(yaml_dir, "en.yml")
    File.open(File.join(RAILS_ROOT, yaml_file), "w") do |file|
      file.write({'en' => @translations}.ya2yaml(:syck_compatible => true))
    end
    print "Wrote new #{yaml_file}\n\n"
  end

  desc "Generates JS bundle i18n files (non-en) and adds them to assets.yml"
  task :generate_js do
    require 'bundler'
    Bundler.setup
    require 'action_controller'
    require 'i18n'
    require 'sexp_processor'
    require 'jammit'
    require 'lib/i18n_extraction/js_extractor.rb'
    I18n.load_path += Dir[Rails.root.join('config', 'locales', '**', '*.{rb,yml}')] +
                      Dir[Rails.root.join('vendor', 'plugins', '*', 'config', 'locales', '**', '*.{rb,yml}')]

    Hash.send :include, HashExtensions

    file_translations = {}

    locales = I18n.available_locales - [:en]
    # allow passing of extra, empty locales by including a comma-separated
    # list of abbreviations in the LOCALES environment variable. e.g.
    #
    # LOCALES=hi,ja,pt,zh-hans rake i18n:generate_js
    locales = locales + ENV['LOCALES'].split(',') if ENV['LOCALES']
    all_translations = I18n.backend.send(:translations)
    flat_translations = all_translations.flatten_keys

    if locales.empty?
      puts "Nothing to do, there are no non-en translations"
      exit 0
    end

    process_files = lambda do |extractor, files, arg_block|
      files.each do |file|
        begin
          extractor.translations = {}
          extractor.process(File.read(file), *arg_block.call(file)) or next

          translations = extractor.translations.flatten_keys.keys
          next if translations.empty?
<commit_msg>Add i18n.reload!, fix compile bug<commit_after>namespace :i18n do
  module HashExtensions
    def flatten_keys(result={}, prefix='')
      each_pair do |k, v|
        if v.is_a?(Hash)
          v.flatten_keys(result, "#{prefix}#{k}.")
        else
          result["#{prefix}#{k}"] = v
        end
      end
      result
    end

    def expand_keys(result = {})
      each_pair do |k, v|
        parts = k.split('.')
        last = parts.pop
        parts.inject(result){ |h, k2| h[k2] ||= {}}[last] = v
      end
      result
    end

    def to_ordered
      keys.sort_by(&:to_s).inject ActiveSupport::OrderedHash.new do |h, k|
        v = fetch(k)
        h[k] = v.is_a?(Hash) ? v.to_ordered : v
        h
      end
    end
  end

  def infer_scope(filename)
    case filename
      when /app\/views\/.*\.handlebars\z/
        filename.gsub(/.*app\/views\/jst\/_?|\.handlebars\z/, '').underscore.gsub(/\/_?/, '.')
      when /app\/controllers\//
        scope = filename.gsub(/.*app\/controllers\/|controller.rb/, '').gsub(/\/_?|_\z/, '.')
        scope == 'application.' ? '' : scope
      when /app\/messages\//
        filename.gsub(/.*app\/|erb/, '').gsub(/\/_?/, '.')
      when /app\/models\//
        scope = filename.gsub(/.*app\/models\/|rb/, '')
        STI_SUPERCLASSES.include?(scope) ? '' : scope
      when /app\/views\//
        filename.gsub(/.*app\/views\/|(html\.|fbml\.)?erb\z/, '').gsub(/\/_?/, '.')
      else
        ''
    end
  end

  desc "Verifies all translation calls"
  task :check => :environment do
    Bundler.require :i18n_tools
    only = if ENV['ONLY']
      ENV['ONLY'].split(',').map{ |path|
        path = '**/' + path if path =~ /\*/
        path = './' + path unless path =~ /\A.?\//
        if path =~ /\*/
          path = Dir.glob(path)
        elsif path !~ /\.(e?rb|js)\z/
          path = Dir.glob(path + '/**/*')
        end
        path
      }.flatten
    end

    STI_SUPERCLASSES = (`grep '^class.*<' ./app/models/*rb|grep -v '::'|sed 's~.*< ~~'|sort|uniq`.split("\n") - ['OpenStruct', 'Tableless']).
      map{ |name| name.underscore + '.' }

    COLOR_ENABLED = ($stdout.tty? rescue false)
    def color(text, color_code)
      COLOR_ENABLED ? "#{color_code}#{text}\e[0m" : text
    end

    def green(text)
      color(text, "\e[32m")
    end

    def red(text)
      color(text, "\e[31m")
    end

    @errors = []
    def process_files(files)
      files.each do |file|
        begin
          print green "." if yield file
        rescue SyntaxError, StandardError
          @errors << "#{$!}\n#{file}"
          print red "F"
        end
      end
    end

    t = Time.now

    I18n.available_locales
    stringifier = proc { |hash, (key, value)|
      hash[key.to_s] = value.is_a?(Hash) ?
        value.inject({}, &stringifier) :
        value
      hash
    }
    @translations = I18n.backend.send(:translations)[:en].inject({}, &stringifier)


    # Ruby
    files = (Dir.glob('./*') - ['./vendor'] + ['./vendor/plugins'] - ['./guard', './tmp']).map { |d| Dir.glob("#{d}/**/*rb") }.flatten.
      reject{ |file| file =~ %r{\A\./(rb-fsevent|vendor/plugins/rails_xss|vendor/plugins/cells|db|spec)/} }
    files &= only if only
    file_count = files.size
    rb_extractor = I18nExtraction::RubyExtractor.new(:translations => @translations)
    process_files(files) do |file|
      source = File.read(file)
      source = Erubis::Eruby.new(source).src if file =~ /\.erb\z/

      sexps = RubyParser.new.parse(source)
      rb_extractor.scope = infer_scope(file)
      rb_extractor.in_html_view = (file =~ /\.(html|facebook)\.erb\z/)
      rb_extractor.process(sexps)
    end


    # JavaScript
    files = (Dir.glob('./public/javascripts/**/*.js') + Dir.glob('./app/views/**/*.erb')).
      reject{ |file| file =~ /\A\.\/public\/javascripts\/(i18nObj.js|i18n.js|jst\/|translations\/|compiled\/handlebars_helpers.js|tinymce\/jscripts\/tiny_mce(.*\/langs|\/tiny_mce\w*\.js))/ }
    files &= only if only
    js_extractor = I18nExtraction::JsExtractor.new(:translations => @translations)
    process_files(files) do |file|
      t2 = Time.now
      ret = js_extractor.process(File.read(file), :erb => (file =~ /\.erb\z/), :filename => file)
      file_count += 1 if ret
      puts "#{file} #{Time.now - t2}" if Time.now - t2 > 1
      ret
    end


    # Handlebars
    files = Dir.glob('./app/views/jst/**/*.handlebars')
    files &= only if only
    handlebars_extractor = I18nExtraction::HandlebarsExtractor.new(:translations => @translations)
    process_files(files) do |file|
      file_count += 1 if handlebars_extractor.process(File.read(file), infer_scope(file))
    end

    print "\n\n"
    failure = @errors.size > 0

    @errors.each_index do |i|
      puts "#{i+1})"
      puts red @errors[i]
      print "\n"
    end

    print "Finished in #{Time.now - t} seconds\n\n"
    total_strings = rb_extractor.total_unique + js_extractor.total_unique + handlebars_extractor.total_unique
    puts send((failure ? :red : :green), "#{file_count} files, #{total_strings} strings, #{@errors.size} failures")
    raise "check command encountered errors" if failure
  end

  desc "Generates a new en.yml file for all translations"
  task :generate => :check do
    yaml_dir = './config/locales/generated'
    FileUtils.mkdir_p(File.join(yaml_dir))
    yaml_file = File.join(yaml_dir, "en.yml")
    File.open(File.join(RAILS_ROOT, yaml_file), "w") do |file|
      file.write({'en' => @translations}.ya2yaml(:syck_compatible => true))
    end
    print "Wrote new #{yaml_file}\n\n"
  end

  desc "Generates JS bundle i18n files (non-en) and adds them to assets.yml"
  task :generate_js do
    require 'bundler'
    Bundler.setup
    require 'action_controller'
    require 'i18n'
    require 'sexp_processor'
    require 'jammit'
    require 'lib/i18n_extraction/js_extractor.rb'
    I18n.load_path += Dir[Rails.root.join('config', 'locales', '**', '*.{rb,yml}')] +
                      Dir[Rails.root.join('vendor', 'plugins', '*', 'config', 'locales', '**', '*.{rb,yml}')]
    I18n.reload!
    Hash.send :include, HashExtensions

    file_translations = {}
    locales = I18n.available_locales - [:en]
    # allow passing of extra, empty locales by including a comma-separated
    # list of abbreviations in the LOCALES environment variable. e.g.
    #
    # LOCALES=hi,ja,pt,zh-hans rake i18n:generate_js
    locales = locales + ENV['LOCALES'].split(',') if ENV['LOCALES']
    all_translations = I18n.backend.send(:translations)
    flat_translations = all_translations.flatten_keys

    if locales.empty?
      puts "Nothing to do, there are no non-en translations"
      exit 0
    end

    process_files = lambda do |extractor, files, arg_block|
      files.each do |file|
        begin
          extractor.translations = {}
          extractor.process(File.read(file), *arg_block.call(file)) or next

          translations = extractor.translations.flatten_keys.keys
          next if translations.empty?

{'fraction_non_alphanumeric': 0.172334459702351, 'fraction_numerical': 0.007333381263929829, 'mean_word_length': 2.4533267130089373, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6917121', 'n_tokens_mistral': 4995, 'n_tokens_neox': 4851, 'n_words': 989}