<commit_before>class TaskVault

  class TaskQueue
    attr_reader :tasks, :retention, :last_id, :interpreters

    def initialize retention: nil, starting_id:-1
      @last_id = starting_id
      @interpreters = {}
      add_interpreter :ruby, Gem.ruby, '.rb'
      @tasks = {
        queued: [],
        ready: [],
        running: [],
        done: []
      }
      self.retention = retention
    end

    def retention= r
      @retention = r.nil? ? nil : BBLib::keep_between(r, 0, nil)
    end

    def next_id
      @last_id+=1
    end

    def queue task
      task = Task.new(**task) if task.is_a?(Hash)
      raise ArgumentError, "Invalid type passed to TaskQueue. Got a '#{task.class}', expected a TaskVault::Task." unless task.is_a?(Task)
      task.status = :queued
      task.id = next_id
      @tasks[:queued].push task
      {id: task.id, name: task.name, start_time:task.start_at}
    end

    def cancel task
      retrieve(task).map do |t|
        t.cancel
        move_task(t, :canceled)
        [t.name, t.status]
      end.to_h
    end

    # Cancels all jobs and clears all completed tasks
    def flush
      tasks.each{ |t| cancel(t) }
      clear
    end

    # Removes all completed tasks
    def clear
      @tasks[:done].clear
    end

    # Removes enough completed tasks to make it down to the retention limit.
    def clean
      if @retention then @tasks[:done].shift until @tasks[:done].size <= @retention end
      nil
    end

    def tasks
      @tasks.map{|k,v| v}.flatten
    end

    def task_list *attributes
      attributes = [:name, :status] if attributes.empty?
      tasks.map do |t|
        attr = Hash.new
        attributes.each{ |a| attr[a.to_sym] = t.send(a.to_sym) if t.respond_to?(a.to_sym) }
        [t.id, attr]
      end.to_h
    end

    def active_tasks
      [@tasks[:queued], @tasks[:ready], @tasks[:running]].flatten
    end

    # Tasks can be retrieved by id, name or with by passing the actual task object.
    # An array containing the above can also be passed.
    def retrieve task
      tasks.find_all{ |t| (task.is_a?(Array) && (task.include?(t.id) || task.include?(t.name) || task.include?(t))) || t.id == task || t.name == task || t == task }
    end

    def status_of id, name: false
      retrieve(id).map{ |t| [(name ? t.name : t.id), t.status] }.to_h
    end

    def value_of id, name: false
      retrieve(id).map{ |t| [(name ? t.name : t.id), t.value] }.to_h
    end

    def add_interpreter name, path, *file_types
      @interpreters[name.to_sym] = {path:path, file_types: file_types}
    end

    def remove_interpreter name
      @interpreters.delete name.to_sym
    end

    STATUSES = {
      queued: :queued,
      ready: :ready,
      running: :running,
      finished: :done,
      error: :done,
      waiting: :queued,
      failed_dependency: :done,
      missing_dependency: :queued,
      timeout: :done,
      canceled: :done,
      unknown: :done
    }

    def get_interpreter inter, script = nil
      return nil if inter.nil? && script.nil?
      return @interpreters[inter][:path] if @interpreters.include?(inter)
      ft = (script.to_s.file_name.chars - script.to_s.file_name(false).chars).join
      return @interpreters.find{|k,v| v[:file_types].include?(ft)}[1][:path]
    end

    def sort
      @tasks[:queued].sort_by!{ |t| [t.priority, t.queued] }
      @tasks[:ready].sort_by!{ |t| [t.priority, t.added] }
      nil
    end

    def move_task task, status
      raise "Invalid status. Cannot move task to '#{status}'." unless STATUSES.include?(status)
      move_to = @tasks[STATUSES[status]]
      tasks = retrieve(task)
      @tasks.map{|k,v| v}.each do |q|
        tasks.each do |t|
          if q.include?(t)
            move_to.push(q.delete(t))
            t.status = status
          end
        end
      end<commit_msg>Added better interpreter checking.<commit_after>class TaskVault

  class TaskQueue
    attr_reader :tasks, :retention, :last_id, :interpreters

    def initialize retention: nil, starting_id:-1
      @last_id = starting_id
      @interpreters = {}
      add_interpreter :ruby, Gem.ruby, '.rb'
      @tasks = {
        queued: [],
        ready: [],
        running: [],
        done: []
      }
      self.retention = retention
    end

    def retention= r
      @retention = r.nil? ? nil : BBLib::keep_between(r, 0, nil)
    end

    def next_id
      @last_id+=1
    end

    def queue task
      task = Task.new(**task) if task.is_a?(Hash)
      raise ArgumentError, "Invalid type passed to TaskQueue. Got a '#{task.class}', expected a TaskVault::Task." unless task.is_a?(Task)
      task.status = :queued
      task.id = next_id
      @tasks[:queued].push task
      {id: task.id, name: task.name, start_time:task.start_at}
    end

    def cancel task
      retrieve(task).map do |t|
        t.cancel
        move_task(t, :canceled)
        [t.name, t.status]
      end.to_h
    end

    # Cancels all jobs and clears all completed tasks
    def flush
      tasks.each{ |t| cancel(t) }
      clear
    end

    # Removes all completed tasks
    def clear
      @tasks[:done].clear
    end

    # Removes enough completed tasks to make it down to the retention limit.
    def clean
      if @retention then @tasks[:done].shift until @tasks[:done].size <= @retention end
      nil
    end

    def tasks
      @tasks.map{|k,v| v}.flatten
    end

    def task_list *attributes
      attributes = [:name, :status] if attributes.empty?
      tasks.map do |t|
        attr = Hash.new
        attributes.each{ |a| attr[a.to_sym] = t.send(a.to_sym) if t.respond_to?(a.to_sym) }
        [t.id, attr]
      end.to_h
    end

    def active_tasks
      [@tasks[:queued], @tasks[:ready], @tasks[:running]].flatten
    end

    # Tasks can be retrieved by id, name or with by passing the actual task object.
    # An array containing the above can also be passed.
    def retrieve task
      tasks.find_all{ |t| (task.is_a?(Array) && (task.include?(t.id) || task.include?(t.name) || task.include?(t))) || t.id == task || t.name == task || t == task }
    end

    def status_of id, name: false
      retrieve(id).map{ |t| [(name ? t.name : t.id), t.status] }.to_h
    end

    def value_of id, name: false
      retrieve(id).map{ |t| [(name ? t.name : t.id), t.value] }.to_h
    end

    def add_interpreter name, path, *file_types
      @interpreters[name.to_sym] = {path:path, file_types: file_types}
    end

    def remove_interpreter name
      @interpreters.delete name.to_sym
    end

    STATUSES = {
      queued: :queued,
      ready: :ready,
      running: :running,
      finished: :done,
      error: :done,
      waiting: :queued,
      failed_dependency: :done,
      missing_dependency: :queued,
      timeout: :done,
      canceled: :done,
      unknown: :done
    }

    def get_interpreter inter, script = nil
      return nil if inter.nil? && script.nil?
      return @interpreters[inter][:path] if @interpreters.include?(inter)
      ft = (script.to_s.file_name.chars - script.to_s.file_name(false).chars).join
      interpreter = @interpreters.find{|k,v| v[:file_types].include?(ft)}
      interpreter[1][:path] rescue nil
    end

    def sort
      @tasks[:queued].sort_by!{ |t| [t.priority, t.queued] }
      @tasks[:ready].sort_by!{ |t| [t.priority, t.added] }
      nil
    end

    def move_task task, status
      raise "Invalid status. Cannot move task to '#{status}'." unless STATUSES.include?(status)
      move_to = @tasks[STATUSES[status]]
      tasks = retrieve(task)
      @tasks.map{|k,v| v}.each do |q|
        tasks.each do |t|
          if q.include?(t)
            move_to.push(q.delete(t))
            t.status = status
          end
        end
      end
{'fraction_non_alphanumeric': 0.12763754045307443, 'fraction_numerical': 0.001035598705501618, 'mean_word_length': 2.317303563761271, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6795798', 'n_tokens_mistral': 2643, 'n_tokens_neox': 2605, 'n_words': 691}