<commit_before>namespace :sheets do

  desc 'Check validity of all sheets'
  task check_validity: :environment do
    project_results = {}

    begin
      current_sheet = 0
      total_sheets = Sheet.current.joins(:project).merge(Project.current).count
      total_projects = Project.current.count
      Project.current.order(:id).each_with_index do |project, project_index|
        project_results[project.id] = { name: project.name, param: project.to_param, invalid_sheets_count: 0, valid_sheets_count: 0 }

        total_project_sheets = project.sheets.count
        project.sheets.order(:id).each_with_index do |sheet, index|
          count_message = " [Sheet #{index + 1} of #{total_project_sheets} (#{"%0.2f" % ((index + 1) * 100.0 / total_project_sheets)}%), Project #{project_index + 1} of #{total_projects}], [All Sheets #{current_sheet + 1} of #{total_sheets} (#{"%0.2f" % ((current_sheet + 1) * 100.0 / total_sheets)}%)]"
          if sheet.successfully_validated?
            print "\r#{"%6d" % sheet.id}:" + " VALID".colorize(:green) + count_message
            project_results[project.id][:valid_sheets_count] += 1
          else
            in_memory_sheet = Validation::InMemorySheet.new(sheet)
            if in_memory_sheet.valid?
              print "\r#{"%6d" % sheet.id}:" + " VALID".colorize(:green) + count_message
              project_results[project.id][:valid_sheets_count] += 1
              sheet.update successfully_validated: true
            else
              puts "\n#{"%6d" % sheet.id}:" + " NOT VALID".colorize(:red) + count_message
              puts "        " + "#{ENV['website_url']}/projects/#{sheet.project.to_param}/sheets/#{sheet.to_param}"
              puts "        " + "#{in_memory_sheet.errors.count} error#{'s' unless in_memory_sheet.errors.count == 1}".colorize(:red)
              in_memory_sheet.errors.each do |error|
                puts "       " + " #{error}"
              end
              project_results[project.id][:invalid_sheets_count] += 1
              sheet.update successfully_validated: false
            end
          end
          current_sheet += 1
        end
      end
    rescue Interrupt
      puts "\nINTERRUPTED".colorize(:red)
    end

    project_results.each do |key, hash|
      puts "\n\n"<commit_msg>Sheet task no longer includes sheets on subjects that are deleted<commit_after>namespace :sheets do

  desc 'Check validity of all sheets'
  task check_validity: :environment do
    project_results = {}

    begin
      current_sheet = 0
      total_sheets = Sheet.current.joins(:project).merge(Project.current).joins(:subject).merge(Subject.current).count
      total_projects = Project.current.count
      Project.current.order(:id).each_with_index do |project, project_index|
        project_results[project.id] = { name: project.name, param: project.to_param, invalid_sheets_count: 0, valid_sheets_count: 0 }

        total_project_sheets = project.sheets.count
        project.sheets.joins(:subject).merge(Subject.current).order(:id).each_with_index do |sheet, index|
          count_message = " [Sheet #{index + 1} of #{total_project_sheets} (#{"%0.2f" % ((index + 1) * 100.0 / total_project_sheets)}%), Project #{project_index + 1} of #{total_projects}], [All Sheets #{current_sheet + 1} of #{total_sheets} (#{"%0.2f" % ((current_sheet + 1) * 100.0 / total_sheets)}%)]"
          if sheet.successfully_validated?
            print "\r#{"%6d" % sheet.id}:" + " VALID".colorize(:green) + count_message
            project_results[project.id][:valid_sheets_count] += 1
          else
            in_memory_sheet = Validation::InMemorySheet.new(sheet)
            if in_memory_sheet.valid?
              print "\r#{"%6d" % sheet.id}:" + " VALID".colorize(:green) + count_message
              project_results[project.id][:valid_sheets_count] += 1
              sheet.update successfully_validated: true
            else
              puts "\n#{"%6d" % sheet.id}:" + " NOT VALID".colorize(:red) + count_message
              puts "        " + "#{ENV['website_url']}/projects/#{sheet.project.to_param}/sheets/#{sheet.to_param}"
              puts "        " + "#{in_memory_sheet.errors.count} error#{'s' unless in_memory_sheet.errors.count == 1}".colorize(:red)
              in_memory_sheet.errors.each do |error|
                puts "       " + " #{error}"
              end
              project_results[project.id][:invalid_sheets_count] += 1
              sheet.update successfully_validated: false
            end
          end
          current_sheet += 1
        end
      end
    rescue Interrupt
      puts "\nINTERRUPTED".colorize(:red)
    end

    project_results.each do |key, hash|
      puts "\n\n"
{'fraction_non_alphanumeric': 0.1331348362398979, 'fraction_numerical': 0.011909825606125054, 'mean_word_length': 2.609363008442057, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3788418', 'n_tokens_mistral': 1487, 'n_tokens_neox': 1428, 'n_words': 277}