<commit_before>#
# Usage :
# rake admin_data:validate_models_bg RAILS_ENV='development' KLASSES='Country,EeeSociety,PhoneNumber,Website,Car,User' TID='20100107095100'
#
require 'fileutils'
namespace :admin_data do
  desc "Run model valiations"
  task :validate_models_bg => :environment do
    usage = %Q{ rake admin_data:validate_models_bg RAILS_ENV='development' KLASSES='Country,EeeSociety,PhoneNumber,Website,Car,User' TID='20100107095100' }
    usage = ' Usage: ' + usage
    tid = ENV['TID']
    raise "tid is blank. #{usage} " if tid.blank?

    klasses = ENV['KLASSES']
    klasses.split(',').each do |klasss|
      puts "processing #{klasss.inspect}"
      klass = AdminData::Util.constantize_klass(klasss)
      name = klass.name
      raise 'name is blank' if name.blank?

      AdminData::Util.write_to_validation_file(tid, 'processing.txt', 'a', name)
      errors = []
      number_of_records = klass.send(:count)
      index = 0
      klass.paginated_each(:order => klass.primary_key) do |record|
        index = index + 1
        s = "processed #{index} of #{number_of_records} #{name} records"
        AdminData::Util.write_to_validation_file(tid, 'processing.txt', 'a', s)
        unless record.valid?
          a = []
          a << klass.name
          a << record.id
          a << record.errors.full_messages
          errors << a
        end
      end
      if errors.any?
        bad_file = File.join(RAILS_ROOT, 'tmp', 'admin_data', 'validate_model', tid , 'bad.txt')
        errors.each { |error| File.open(bad_file, 'a') {|f| f.puts(error.join(' | ')) } }
      else
        good_file = File.join(RAILS_ROOT, 'tmp', 'admin_data', 'validate_model', tid , 'good.txt')
        s = "#{klass.name} | #{klass.send(:count)} records: all valid"
        File.open(good_file, 'a') {|f| f.puts(s)}
      end
    end

    done_file = File.join(RAILS_ROOT, 'tmp', 'admin_data', 'validate_model', tid , 'done.txt')
    File.open(done_file, 'a') {|f| f.puts('') }
  end
end<commit_msg>wrap the whole rake task in a rescue block and write the exception info to a file<commit_after>#
# Usage :
# rake admin_data:validate_models_bg RAILS_ENV='development' KLASSES='Country,EeeSociety,PhoneNumber,Website,Car,User' TID='20100107095100'
#
require 'fileutils'
namespace :admin_data do
  desc "Run model valiations"
  task :validate_models_bg => :environment do
    begin
      usage = %Q{ rake admin_data:validate_models_bg RAILS_ENV='development' KLASSES='Country,EeeSociety,PhoneNumber,Website,Car,User' TID='20100107095100' }
      usage = ' Usage: ' + usage
      tid = ENV['TID']
      raise "tid is blank. #{usage} " if tid.blank?

      klasses = ENV['KLASSES']
      klasses.split(',').each do |klasss|
        puts "processing #{klasss.inspect}"
        klass = AdminData::Util.constantize_klass(klasss)
        name = klass.name
        raise 'name is blank' if name.blank?

        AdminData::Util.write_to_validation_file(tid, 'processing.txt', 'a', name)
        errors = []
        number_of_records = klass.send(:count)
        index = 0
        klass.paginated_each(:order => klass.primary_key) do |record|
          index = index + 1
          s = "processed #{index} of #{number_of_records} #{name} records"
          AdminData::Util.write_to_validation_file(tid, 'processing.txt', 'a', s)
          unless record.valid?
            a = []
            a << klass.name
            a << record.id
            a << record.errors.full_messages
            errors << a
          end
        end
        if errors.any?
          bad_file = File.join(RAILS_ROOT, 'tmp', 'admin_data', 'validate_model', tid, 'bad.txt')
          errors.each { |error| File.open(bad_file, 'a') {|f| f.puts(error.join(' | ')) } }
        else
          good_file = File.join(RAILS_ROOT, 'tmp', 'admin_data', 'validate_model', tid, 'good.txt')
          s = "#{klass.name} | #{klass.send(:count)} records: all valid"
          File.open(good_file, 'a') {|f| f.puts(s)}
        end
      end

      done_file = File.join(RAILS_ROOT, 'tmp', 'admin_data', 'validate_model', tid, 'done.txt')
      File.open(done_file, 'a') {|f| f.puts('') }
    rescue Exception => e
      s = AdminData::Util.exception_info(e)
      AdminData::Util.write_to_validation_file(nil, 'rake_errors.txt', 'a', s)
    end
  end
end
{'fraction_non_alphanumeric': 0.1444186046511628, 'fraction_numerical': 0.013953488372093023, 'mean_word_length': 2.967712177121771, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '751195', 'n_tokens_mistral': 1518, 'n_tokens_neox': 1466, 'n_words': 316}