<commit_before>require 'tmpdir'

curr_dir = Dir.pwd
tmp_dir = Dir.tmpdir
FileUtils.rm_rf "#{tmp_dir}/rails-i18n" if File.exists? "#{tmp_dir}/rails-i18n"

exec %(
  cd #{tmp_dir};
  mkdir rails-i18n;
  cd rails-i18n;
  echo 'created tmp dir';
  echo 'cloning rails';
  git clone git://github.com/rails/rails.git;
  cp ./rails/actionpack/lib/action_view/locale/en.yml #{curr_dir}/rails/rails/action_view.yml;
  cp ./rails/activerecord/lib/active_record/locale/en.yml #{curr_dir}/rails/rails/active_record.yml;
  cp ./rails/activesupport/lib/active_support/locale/en.yml #{curr_dir}/rails/rails/active_support.yml;
  cd ..;
  rm -rf rails-i18n;
)

<commit_msg>A better locale fetching script, using only curl.<commit_after>curr_dir = File.expand_path(File.dirname(__FILE__))
rails_locale_dir = File.expand_path(File.join(curr_dir, "..", "rails"))

puts "Fetching latest Rails locale files to #{rails_locale_dir}"

exec %(
  curl http://github.com/rails/rails/tree/master/actionpack/lib/action_view/locale/en.yml?raw=true > #{rails_locale_dir}/action_view.yml

  curl http://github.com/rails/rails/tree/master/activerecord/lib/active_record/locale/en.yml?raw=true > #{rails_locale_dir}/active_record.yml

  curl http://github.com/rails/rails/tree/master/activesupport/lib/active_support/locale/en.yml?raw=true > #{rails_locale_dir}/active_support.yml
)

{'fraction_non_alphanumeric': 0.15270570793180133, 'fraction_numerical': 0.007412898443291327, 'mean_word_length': 4.921052631578948, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2061634', 'n_tokens_mistral': 545, 'n_tokens_neox': 492, 'n_words': 71}