<commit_before>require File.expand_path("../../config/boot.rb", __FILE__)
require File.expand_path("../../app/helpers/states.rb", __FILE__)
require File.expand_path("../../app/helpers/colorize.rb", __FILE__)

namespace :'congress-forms' do
  namespace :'delayed_job' do
    desc "perform all fills on the Delayed::Job error_or_failure queue, captchad fills first, optionally provide bioguide regex or job id"
    task :perform_fills, :regex, :job_id, :overrides do |t, args|
      require 'pp'

      regex = args[:regex].blank? ? nil : Regexp.compile(args[:regex])
      overrides = args[:overrides].blank? ? {} : eval(args[:overrides])

      jobs = retrieve_jobs args

      captcha_jobs = []
      noncaptcha_jobs = []

      cm_hash = build_cm_hash
      captcha_hash = build_captcha_hash

      jobs.each do |job|
        cm_id, = congress_member_id_and_args_from_handler(job.handler)
        cm = retrieve_congress_member_cached(cm_hash, cm_id)
        if regex.nil? or regex.match(cm.bioguide_id)
          if retrieve_captchad_cached(captcha_hash, cm.id)
            captcha_jobs.push job
          else
            noncaptcha_jobs.push job
          end
        end
      end
      captcha_jobs.each do |job|
        begin
          cm_id, cm_args = congress_member_id_and_args_from_handler(job.handler)
          cm = retrieve_congress_member_cached(cm_hash, cm_id)
          puts red("Job #" + job.id.to_s + ", bioguide " + cm.bioguide_id)
          pp cm_args
          result = cm.fill_out_form cm_args[0].merge(overrides), cm_args[1] do |img|
            puts img
            STDIN.gets.strip
          end
        rescue
        end
        job.destroy
      end
      noncaptcha_jobs.each do |job|
        begin
          cm_id, cm_args = congress_member_id_and_args_from_handler(job.handler)
          cm = retrieve_congress_member_cached(cm_hash, cm_id)
          puts red("Job #" + job.id.to_s + ", bioguide " + cm.bioguide_id)
          pp cm_args
          result = cm.fill_out_form cm_args[0].merge(overrides), cm_args[1]
        rescue
        end
        job.destroy
      end
    end
    desc "override a field on the Delayed::Job error_or_failure queue, optionally provide bioguide regex or job id"
    task :override_field, :regex, :job_id, :overrides do |t, args|
      regex = args[:regex].blank? ? nil : Regexp.compile(args[:regex])
      overrides = args[:overrides].blank? ? {} : eval(args[:overrides])

      jobs = retrieve_jobs args

      if job_id.nil?
        jobs = Delayed::Job.where(queue: "error_or_failure")
      else
        jobs = [Delayed::Job.find(job_id)]
      end

      cm_hash = build_cm_hash
      captcha_hash = build_captcha_hash

      jobs.each do |job|
        cm_id, = congress_member_id_and_args_from_handler(job.handler)
        cm = retrieve_congress_member_cached(cm_hash, cm_id)

        if regex.nil? or regex.match(cm.bioguide_id)
          handler = YAML.load job.handler
          handler.args[0] = handler.args[0].merge(overrides)
          job.handler = YAML.dump handler
          job.save
        end
      end
    end
    desc "destroy all fills on the Delayed::Job error_or_failure queue provided a specific bioguide or job_id"
    task :destroy_fills, :bioguide, :job_id do |t, args|
      cm = CongressMember.bioguide(args[:bioguide])

      jobs = retrieve_jobs args

      jobs.each do |job|
        cm_id, = congress_member_id_and_args_from_handler(job.handler)
        if not args[:job_id].nil? or cm_id.to_i == cm.id
          puts red("Destroying job #" + job.id.to_s)
          job.destroy
        end
      end
    end
    desc "calculate # of jobs per member on the Delayed::Job error_or_failure queue"
    task :jobs_per_member do |t, args|
      jobs = Delayed::Job.where(queue: "error_or_failure")

      people = {}
      cm_hash = build_cm_hash

      jobs.each do |job|
        cm_id, = congress_member_id_and_args_from_handler(job.handler)
        cm = retrieve_congress_member_cached(cm_hash, cm_id)
        if people.keys.include? cm.bioguide_id
          people[cm.bioguide_id] += 1
        else
          people[cm.bioguide_id] = 1
        end
      end
      captchad_hash = {}
      total_captchad_jobs = 0
      total_jobs = 0
      people.each do |k, v|
        if CongressMember.bioguide(k).has_captcha?
          captchad_hash[k] = true
          total_captchad_jobs += v
        end
        total_jobs += v
      end
      puts "The captcha'd congress members are displayed in red.\n\n"
      people.sort_by { |k, v| v}.reverse.each do |k, v|
        key_colored = captchad_hash[k] ? red(k) : k
        puts key_colored + ": " + v.to_s
      end
      puts "\nTotal jobs: "+total_jobs.to_s
      puts "Total captcha'd jobs: "+total_captchad_jobs.to_s<commit_msg>Provide a few additional useful stats in jobs_per_member<commit_after>require File.expand_path("../../config/boot.rb", __FILE__)
require File.expand_path("../../app/helpers/states.rb", __FILE__)
require File.expand_path("../../app/helpers/colorize.rb", __FILE__)

namespace :'congress-forms' do
  namespace :'delayed_job' do
    desc "perform all fills on the Delayed::Job error_or_failure queue, captchad fills first, optionally provide bioguide regex or job id"
    task :perform_fills, :regex, :job_id, :overrides do |t, args|
      require 'pp'

      regex = args[:regex].blank? ? nil : Regexp.compile(args[:regex])
      overrides = args[:overrides].blank? ? {} : eval(args[:overrides])

      jobs = retrieve_jobs args

      captcha_jobs = []
      noncaptcha_jobs = []

      cm_hash = build_cm_hash
      captcha_hash = build_captcha_hash

      jobs.each do |job|
        cm_id, = congress_member_id_and_args_from_handler(job.handler)
        cm = retrieve_congress_member_cached(cm_hash, cm_id)
        if regex.nil? or regex.match(cm.bioguide_id)
          if retrieve_captchad_cached(captcha_hash, cm.id)
            captcha_jobs.push job
          else
            noncaptcha_jobs.push job
          end
        end
      end
      captcha_jobs.each do |job|
        begin
          cm_id, cm_args = congress_member_id_and_args_from_handler(job.handler)
          cm = retrieve_congress_member_cached(cm_hash, cm_id)
          puts red("Job #" + job.id.to_s + ", bioguide " + cm.bioguide_id)
          pp cm_args
          result = cm.fill_out_form cm_args[0].merge(overrides), cm_args[1] do |img|
            puts img
            STDIN.gets.strip
          end
        rescue
        end
        job.destroy
      end
      noncaptcha_jobs.each do |job|
        begin
          cm_id, cm_args = congress_member_id_and_args_from_handler(job.handler)
          cm = retrieve_congress_member_cached(cm_hash, cm_id)
          puts red("Job #" + job.id.to_s + ", bioguide " + cm.bioguide_id)
          pp cm_args
          result = cm.fill_out_form cm_args[0].merge(overrides), cm_args[1]
        rescue
        end
        job.destroy
      end
    end
    desc "override a field on the Delayed::Job error_or_failure queue, optionally provide bioguide regex or job id"
    task :override_field, :regex, :job_id, :overrides do |t, args|
      regex = args[:regex].blank? ? nil : Regexp.compile(args[:regex])
      overrides = args[:overrides].blank? ? {} : eval(args[:overrides])

      jobs = retrieve_jobs args

      if job_id.nil?
        jobs = Delayed::Job.where(queue: "error_or_failure")
      else
        jobs = [Delayed::Job.find(job_id)]
      end

      cm_hash = build_cm_hash
      captcha_hash = build_captcha_hash

      jobs.each do |job|
        cm_id, = congress_member_id_and_args_from_handler(job.handler)
        cm = retrieve_congress_member_cached(cm_hash, cm_id)

        if regex.nil? or regex.match(cm.bioguide_id)
          handler = YAML.load job.handler
          handler.args[0] = handler.args[0].merge(overrides)
          job.handler = YAML.dump handler
          job.save
        end
      end
    end
    desc "destroy all fills on the Delayed::Job error_or_failure queue provided a specific bioguide or job_id"
    task :destroy_fills, :bioguide, :job_id do |t, args|
      cm = CongressMember.bioguide(args[:bioguide])

      jobs = retrieve_jobs args

      jobs.each do |job|
        cm_id, = congress_member_id_and_args_from_handler(job.handler)
        if not args[:job_id].nil? or cm_id.to_i == cm.id
          puts red("Destroying job #" + job.id.to_s)
          job.destroy
        end
      end
    end
    desc "calculate # of jobs per member on the Delayed::Job error_or_failure queue"
    task :jobs_per_member do |t, args|
      jobs = Delayed::Job.where(queue: "error_or_failure")

      people = {}
      cm_hash = build_cm_hash

      jobs.each do |job|
        cm_id, = congress_member_id_and_args_from_handler(job.handler)
        cm = retrieve_congress_member_cached(cm_hash, cm_id)
        if people.keys.include? cm.bioguide_id
          people[cm.bioguide_id] += 1
        else
          people[cm.bioguide_id] = 1
        end
      end
      captchad_hash = {}
      total_captchad_jobs = 0
      total_jobs = 0
      people.each do |k, v|
        if CongressMember.bioguide(k).has_captcha?
          captchad_hash[k] = true
          total_captchad_jobs += v
        end
        total_jobs += v
      end
      puts "The captcha'd congress members are displayed in red.\n\n"
      people.sort_by { |k, v| v}.reverse.each do |k, v|
        key_colored = captchad_hash[k] ? red(k) : k
        puts key_colored + ": " + v.to_s
      end
      puts "\nTotal jobs: "+total_jobs.to_s
      puts "Total captcha'd jobs: "+total_captchad_jobs.to_s
      puts "\nTotal members: "+people.length.to_s
      puts "Total captcha'd members: "+captchad_hash.length.to_s
{'fraction_non_alphanumeric': 0.09822534048699959, 'fraction_numerical': 0.0020635575732562937, 'mean_word_length': 2.4396735273243437, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5417509', 'n_tokens_mistral': 3335, 'n_tokens_neox': 3248, 'n_words': 713}