<commit_before>module Mobilize
  module Jobtracker
    #adds convenience methods
    require "#{File.dirname(__FILE__)}/helpers/jobtracker_helper"

    def Jobtracker.max_run_time_workers
      #return workers who have been cranking away for 6+ hours
        workers = Jobtracker.workers('working').select do |w|
            w.job['run_at'].to_s.length>0 and 
              (Time.now.utc - Time.parse(w.job['run_at'])) > Jobtracker.max_run_time
        end
        return workers
    end

    def Jobtracker.start_worker(count=nil)
      Resque.start_workers(count)
    end

    def Jobtracker.kill_workers(count=nil)
      Resque.kill_workers(count)
    end

    def Jobtracker.run_notifications
      if Jobtracker.notif_due?
        notifs = []
        if Jobtracker.failures.length>0
          failure_hash = Resque.new_failures_by_email
          failure_hash.each do |email,stage_paths|
            n = {}
            n['subject'] = "#{stage_paths.keys.length.to_s} new failed jobs, #{stage_paths.values.map{|v| v.values}.flatten.sum.to_s} failures"
            #one row per exception type, with the job name
            n['body'] = stage_paths.map do |path,exceptions| 
                                          exceptions.map do |exc_to_s,times| 
                                            [path," : ",exc_to_s,", ",times," times"].join
                                          end
                                        end.flatten.join("\n\n")
            u = User.where(:name=>email.split("@").first).first
            if u
              runner_dst = Dataset.find_by_url("gsheet://#{u.runner.path}")
              n['body'] += "\n\n#{runner_dst.http_url}" if runner_dst and runner_dst.http_url
            end
            n['to'] = email
            n['bcc'] = [Gdrive.admin_group_name,Gdrive.domain].join("@")
            notifs << n
          end
        end
        lws = Jobtracker.max_run_time_workers<commit_msg>removed bcc to admins for user emails<commit_after>module Mobilize
  module Jobtracker
    #adds convenience methods
    require "#{File.dirname(__FILE__)}/helpers/jobtracker_helper"

    def Jobtracker.max_run_time_workers
      #return workers who have been cranking away for 6+ hours
        workers = Jobtracker.workers('working').select do |w|
            w.job['run_at'].to_s.length>0 and 
              (Time.now.utc - Time.parse(w.job['run_at'])) > Jobtracker.max_run_time
        end
        return workers
    end

    def Jobtracker.start_worker(count=nil)
      Resque.start_workers(count)
    end

    def Jobtracker.kill_workers(count=nil)
      Resque.kill_workers(count)
    end

    def Jobtracker.run_notifications
      if Jobtracker.notif_due?
        notifs = []
        if Jobtracker.failures.length>0
          failure_hash = Resque.new_failures_by_email
          failure_hash.each do |email,stage_paths|
            n = {}
            n['subject'] = "#{stage_paths.keys.length.to_s} new failed jobs, #{stage_paths.values.map{|v| v.values}.flatten.sum.to_s} failures"
            #one row per exception type, with the job name
            n['body'] = stage_paths.map do |path,exceptions| 
                                          exceptions.map do |exc_to_s,times| 
                                            [path," : ",exc_to_s,", ",times," times"].join
                                          end
                                        end.flatten.join("\n\n")
            u = User.where(:name=>email.split("@").first).first
            if u
              runner_dst = Dataset.find_by_url("gsheet://#{u.runner.path}")
              n['body'] += "\n\n#{runner_dst.http_url}" if runner_dst and runner_dst.http_url
            end
            n['to'] = email
            #uncomment to receive a copy of emails sent to users
            #n['bcc'] = [Gdrive.admin_group_name,Gdrive.domain].join("@")
            notifs << n
          end
        end
        lws = Jobtracker.max_run_time_workers
{'fraction_non_alphanumeric': 0.10976226605968639, 'fraction_numerical': 0.0015174506828528073, 'mean_word_length': 1.8473722102231822, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2207207', 'n_tokens_mistral': 1174, 'n_tokens_neox': 1162, 'n_words': 231}