<commit_before>namespace :citizen_budget_model do
  desc 'Pulls out strings marked for translations'
  task translations: :environment do
    require 'citizen_budget_model/gettext_helper'
    require 'citizen_budget_model/js_parser'

    GetText::Tools::XGetText.add_parser(CitizenBudgetModel::JsParser)
    keys = CitizenBudgetModel::GetTextHelper.messages('app/**/*.{erb,rb,js}')

    # Unless the backend already has a key, add a default to the default locale.
    data = {}
    keys.each do |key|
      unless I18n.backend.exists?(I18n.default_locale, key)
        data[key] = key
      end
    end

    unless data.empty?
      if ENV['CONFIRM']
        puts "Adding:"
        puts data.keys.sort
        puts 'Continue? (y/n)'
        unless STDIN.gets == "y\n"
          next
        end
      end

      I18n.backend.store_translations(I18n.default_locale, data, escape: false)
      puts "#{data.size} added"
    end

    # List all the keys to be deleted.
    delete = []
    store = I18n.backend.backends.first.store
    store.keys.each do |k|
      key = k.split('.', 2)[1]
      unless keys.include?(key)
        delete << k
      end
    end

    # Delete keys that are no longer extant.
    unless delete.empty?
      if ENV['CONFIRM']
        puts "Deleting:"
        puts delete.sort
        puts 'Continue? (y/n)'
        unless STDIN.gets == "y\n"
          next
        end
      end

      number = store.del(delete)
      puts "#{number} deleted"
    end
  end
end<commit_msg>refactor rake task for brevity<commit_after>namespace :citizen_budget_model do
  desc 'Pulls out strings marked for translations'
  task translations: :environment do
    require 'citizen_budget_model/gettext_helper'
    require 'citizen_budget_model/js_parser'

    def confirm(message, objects)
      unless objects.empty?
        if ENV['CONFIRM']
          puts message
          puts objects.sort
          puts 'Continue? (y/n)'
          unless STDIN.gets == "y\n"
            return
          end
        end
        yield
      end
    end

    GetText::Tools::XGetText.add_parser(CitizenBudgetModel::JsParser)

    keys = CitizenBudgetModel::GetTextHelper.messages('app/**/*.{erb,rb,js}')

    # Unless the backend already has a key, add a default to the default locale.
    keys_to_add = keys.reject do |key|
      I18n.backend.exists?(I18n.default_locale, key)
    end

    store = I18n.backend.backends.first.store

    # Delete keys that are no longer extant.
    keys_to_delete = store.keys.reject do |k|
      keys.include?(k.split('.', 2)[1])
    end

    confirm('Adding:', keys_to_add) do
      I18n.backend.store_translations(I18n.default_locale, Hash[keys_to_add.zip(keys_to_add)], escape: false)
      puts "#{data.size} added"

      confirm('Deleting:', keys_to_delete) do
        number = store.del(keys_to_delete)
        puts "#{number} deleted"
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.10420979986197378, 'fraction_numerical': 0.008281573498964804, 'mean_word_length': 2.4969843184559712, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1194106', 'n_tokens_mistral': 971, 'n_tokens_neox': 937, 'n_words': 231}