<commit_before>#!/usr/bin/env ruby
require 'fileutils'

# Useful values
RAILS_REPO_LOCATION = 'git://github.com/rails/rails.git'
RAILS_CHECKOUT      = 'rails'
BASE_DIR            = File.expand_path(File.dirname(__FILE__))
OUTPUT_DIR          = File.join(BASE_DIR, 'output')

# Utility functions
def check_out_clean_rails_repo
  puts "Checking out clean rails repo from #{RAILS_REPO_LOCATION} into #{RAILS_CHECKOUT}."
  FileUtils.rm_r(RAILS_CHECKOUT) if Dir.exists?(RAILS_CHECKOUT)
  `git clone #{RAILS_REPO_LOCATION} #{RAILS_CHECKOUT}`
end

def clobber_output_dir
  puts "Cleaning up old output."
  FileUtils.rm_r(OUTPUT_DIR) if Dir.exists?(OUTPUT_DIR)
  FileUtils.mkdir(OUTPUT_DIR)
end


# Processing
check_out_clean_rails_repo
clobber_output_dir

FileUtils.cd('rails') do
  all_tags = `git tag`.split("\n")
  released_version_regex = 'v3\.2\.\d+'
  released_version_tags = all_tags.select { |t|
                            t.match(/^#{released_version_regex}$/)
                          }

  released_version_tags.each do |tag|
    tag_output_directory = File.join(OUTPUT_DIR, tag)

    puts "Checking out Rails #{tag}"
    `git checkout #{tag} &> /dev/null`
    if File.exists?('Gemfile')
      puts "  Gemfile found. Bundling..."
      `bundle`
      puts "  Bundling complete. Generating guides..."
      FileUtils.cd('railties') do
        `bundle exec rake generate_guides ALL=1 RAILS_VERSION=#{tag} &> /dev/null`
        puts "  Generating guides complete. Copying to #{tag_output_directory}..."
        `cp -R guides/output/ #{tag_output_directory}`
      end
      puts "  Copying complete."
    else
      puts "  No Gemfile. I quit."
    end
    puts "Done processing #{tag}."
  end
end


# For each tag of a released version after 2.3.x
# and 'edge' too? edge needs a commit hash
<commit_msg>Keep track of which versions were generated successfully and which failed<commit_after>#!/usr/bin/env ruby
require 'fileutils'

# Useful values
RAILS_REPO_LOCATION = 'git://github.com/rails/rails.git'
RAILS_CHECKOUT      = 'rails'
BASE_DIR            = File.expand_path(File.dirname(__FILE__))
OUTPUT_DIR          = File.join(BASE_DIR, 'output')

# Utility functions
def check_out_clean_rails_repo
  puts "Checking out clean rails repo from #{RAILS_REPO_LOCATION} into #{RAILS_CHECKOUT}."
  FileUtils.rm_r(RAILS_CHECKOUT) if Dir.exists?(RAILS_CHECKOUT)
  `git clone #{RAILS_REPO_LOCATION} #{RAILS_CHECKOUT}`
end

def clobber_output_dir
  puts "Cleaning up old output."
  FileUtils.rm_r(OUTPUT_DIR) if Dir.exists?(OUTPUT_DIR)
  FileUtils.mkdir(OUTPUT_DIR)
end


# Processing
check_out_clean_rails_repo
clobber_output_dir

FileUtils.cd('rails') do
  all_tags = `git tag`.split("\n")
  released_version_regex = 'v3\.2\.\d+'
  released_version_tags = all_tags.select { |t|
                            t.match(/^#{released_version_regex}$/)
                          }

  successfully_generated = []
  failed_to_generate     = []

  released_version_tags.each do |tag|
    tag_output_directory = File.join(OUTPUT_DIR, tag)

    puts "Checking out Rails #{tag}"
    `git checkout #{tag} &> /dev/null`
    if File.exists?('Gemfile')
      puts "  Gemfile found. Bundling..."
      `bundle`
      puts "  Bundling complete. Generating guides..."
      FileUtils.cd('railties') do
        `bundle exec rake generate_guides ALL=1 RAILS_VERSION=#{tag} &> /dev/null`
        puts "  Generating guides complete. Copying to #{tag_output_directory}..."
        `cp -R guides/output/ #{tag_output_directory}`
      end
      puts "  Copying complete."
    else
      puts "  No Gemfile. I quit."
    end
    if File.exists?(File.join(tag_output_directory, 'index.html'))
      successfully_generated << tag
    else
      failed_to_generate << tag
    end
    puts "Done processing #{tag}."
  end
end


# For each tag of a released version after 2.3.x
# and 'edge' too? edge needs a commit hash

{'fraction_non_alphanumeric': 0.11402831402831402, 'fraction_numerical': 0.002574002574002574, 'mean_word_length': 3.1517094017094016, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '326856', 'n_tokens_mistral': 1337, 'n_tokens_neox': 1330, 'n_words': 309}