<commit_before>require 'pp'

namespace :fixer do

#########################################################################################################
  desc "check fixer status mismatches"
  task check: [:environment] do

    verbose = ENV['VERBOSE']
    debug   = ENV['DEBUG']
    limit   = ENV['LIMIT']
    ok_to_recover = ENV['RECOVER']
    recover_types = Hash[ ENV['RECOVER_TYPES'] ? ENV['RECOVER_TYPES'].split(/\ *,\ */).map{|t| [t, true]} : [] ]
    debug and pp recover_types.inspect
    puts "Finding all tasks with status mismatch..."
    puts "Will try to recover these types: #{ recover_types.keys.inspect }"
    mismatched_tasks = Task.get_mismatched_status('working')
    mismatched_report = Hash.new{ |h,k| h[k] = 0 }
    mismatched_tasks.each do |task|
      mismatched_report[task.type] += 1
      if limit and limit.to_i <= mismatched_report[task.type]
        debug and puts "Limit #{limit} reached for #{task.type}. Skipping eval of task #{task.id}"
        next
      end
      debug and puts "Task #{task.type} #{task.id} has status '#{task.status}' with results: " + task.results.inspect
      if recover_types.has_key?(task.type) and ok_to_recover
        verbose and puts "Calling #{task.type}.find(#{task.id}).recover!"
        task.recover!
        verbose and puts "#{task.type}.find(#{task.id}) new status == #{task.status}"
        mismatched_report[task.status] += 1
      end
    end

    puts "These tasks with mismatched status were found:"
    pp mismatched_report
  end

#########################################################################################################
  desc "nudge unfinished tasks toward the finish line"
  task nudge: [:environment] do

    verbose = ENV['VERBOSE']
    debug   = ENV['DEBUG']
    limit   = ENV['LIMIT']
    ok_to_recover = ENV['RECOVER']
    recover_types = Hash[ ENV['RECOVER_TYPES'] ? ENV['RECOVER_TYPES'].split(/\ *,\ */).map{|t| [t, true]} : [] ]
    report     = Hash.new{ |h,k| h[k] = 0 }
    unfinished = Task.incomplete
    verbose and puts "Nudging #{unfinished.count} unfinished tasks"
    verbose and puts "Will try to recover these types: #{ recover_types.keys.inspect }"
    unfinished.find_in_batches do |taskgroup|
      taskgroup.each do |task|
        if limit and limit.to_i <= report[task.type]
          next
        end

        debug and puts "Task.find(#{task.id}) -> #{task.type}"
        report[task.type] += 1

        if task.stuck?
          report[task.type+'-stuck'] += 1
          if ok_to_recover and recover_types.has_key?(task.type)
            begin
              task.recover!
              report[task.type+'-recovered'] += 1
            rescue Exceptions::PrivateFileNotFound => err
              STDERR.puts err  # warn and continue
            rescue => err
              raise err # re-throw
            end
          end
        end
      end
    end

    verbose and pp report
<commit_msg>after nudging a stuck task, tell the task owner to check all tasks (to re-create if recover ended in cancel)<commit_after>require 'pp'

namespace :fixer do

#########################################################################################################
  desc "check fixer status mismatches"
  task check: [:environment] do

    verbose = ENV['VERBOSE']
    debug   = ENV['DEBUG']
    limit   = ENV['LIMIT']
    ok_to_recover = ENV['RECOVER']
    recover_types = Hash[ ENV['RECOVER_TYPES'] ? ENV['RECOVER_TYPES'].split(/\ *,\ */).map{|t| [t, true]} : [] ]
    debug and pp recover_types.inspect
    puts "Finding all tasks with status mismatch..."
    puts "Will try to recover these types: #{ recover_types.keys.inspect }"
    mismatched_tasks = Task.get_mismatched_status('working')
    mismatched_report = Hash.new{ |h,k| h[k] = 0 }
    mismatched_tasks.each do |task|
      mismatched_report[task.type] += 1
      if limit and limit.to_i <= mismatched_report[task.type]
        debug and puts "Limit #{limit} reached for #{task.type}. Skipping eval of task #{task.id}"
        next
      end
      debug and puts "Task #{task.type} #{task.id} has status '#{task.status}' with results: " + task.results.inspect
      if recover_types.has_key?(task.type) and ok_to_recover
        verbose and puts "Calling #{task.type}.find(#{task.id}).recover!"
        task.recover!
        verbose and puts "#{task.type}.find(#{task.id}) new status == #{task.status}"
        mismatched_report[task.status] += 1
      end
    end

    puts "These tasks with mismatched status were found:"
    pp mismatched_report
  end

#########################################################################################################
  desc "nudge unfinished tasks toward the finish line"
  task nudge: [:environment] do

    verbose = ENV['VERBOSE']
    debug   = ENV['DEBUG']
    limit   = ENV['LIMIT']
    ok_to_recover = ENV['RECOVER']
    recover_types = Hash[ ENV['RECOVER_TYPES'] ? ENV['RECOVER_TYPES'].split(/\ *,\ */).map{|t| [t, true]} : [] ]
    report     = Hash.new{ |h,k| h[k] = 0 }
    unfinished = Task.incomplete
    verbose and puts "Nudging #{unfinished.count} unfinished tasks"
    verbose and puts "Will try to recover these types: #{ recover_types.keys.inspect }"
    unfinished.find_in_batches do |taskgroup|
      taskgroup.each do |task|
        if limit and limit.to_i <= report[task.type]
          next
        end

        debug and puts "Task.find(#{task.id}) -> #{task.type}"
        report[task.type] += 1

        if task.stuck?
          report[task.type+'-stuck'] += 1
          if ok_to_recover and recover_types.has_key?(task.type)
            begin
              task.recover!
              task.owner.check_tasks
              report[task.type+'-recovered'] += 1
            rescue Exceptions::PrivateFileNotFound => err
              STDERR.puts err  # warn and continue
            rescue => err
              raise err # re-throw
            end
          end
        end
      end
    end

    verbose and pp report

{'fraction_non_alphanumeric': 0.18726968174204356, 'fraction_numerical': 0.0023450586264656616, 'mean_word_length': 2.7839036755386566, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '878238', 'n_tokens_mistral': 1836, 'n_tokens_neox': 1795, 'n_words': 478}