<commit_before>require 'jobba/clause'
require 'jobba/id_clause'
require 'jobba/clause_factory'

class Jobba::Query

  include Jobba::Common

  def where(options)
    options.each do |kk,vv|
      clauses.push(Jobba::ClauseFactory.new_clause(kk,vv))
    end

    self
  end

  def count
    run_with_block(&COUNT_STATUSES)
  end

  def empty?
    count == 0
  end

  # At the end of a chain of `where`s, the user will call methods that expect
  # to run on the result of the executed `where`s.  So if we don't know what
  # the method is, execute the `where`s and pass the method to its output.

  def method_missing(method_name, *args, &block)
    if Jobba::Statuses.instance_methods.include?(method_name)
      run.send(method_name, *args, &block)
    else
      super
    end
  end

  def respond_to?(method_name)
    Jobba::Statuses.instance_methods.include?(method_name) || super
  end

  def run
    run_with_block(&GET_STATUSES)
  end

  protected

  attr_accessor :clauses

  def initialize
    @clauses = []
  end

  GET_STATUSES = ->(working_set) {
    ids = Jobba.redis.zrange(working_set, 0, -1)
    Jobba::Statuses.new(ids)
  }

  COUNT_STATUSES = ->(working_set) {
    Jobba.redis.zcard(working_set)
  }

  def run_with_block(&working_set_block)

    # TODO I think this code can be rewritten to send all calls to redis
    # in one or maybe two `multi` blocks.  A complication comes in
    # how the GET_STATUSES block uses the result of a redis call in
    # its instantiation of a Statuses object.  That would have to
    # be split apart somehow -- like maybe this method can take two blocks
    # -- one that is run inside the multi block, and another that runs
    # outside of the multi block taking as input the output of the first
    # block.

    load_default_clause if clauses.empty?
    working_set = nil

    clauses.each_with_index do |clause, ii|
      clause_set = clause.to_new_set

      if working_set.nil?
        working_set = clause_set
      else
        redis.zinterstore(working_set, [working_set, clause_set], weights: [0, 0])
        redis.del(clause_set)
      end
    end

    working_set_block.call(working_set).tap do
      redis.del(working_set)
    end
  end

  def load_default_clause
    where(state: Jobba::State::ALL.collect(&:name))
  end

end<commit_msg>queries now run in one redis call\!<commit_after>require 'jobba/clause'
require 'jobba/id_clause'
require 'jobba/clause_factory'

class Jobba::Query

  include Jobba::Common

  def where(options)
    options.each do |kk,vv|
      clauses.push(Jobba::ClauseFactory.new_clause(kk,vv))
    end

    self
  end

  def count
    _run(COUNT_STATUSES)
  end

  def empty?
    count == 0
  end

  # At the end of a chain of `where`s, the user will call methods that expect
  # to run on the result of the executed `where`s.  So if we don't know what
  # the method is, execute the `where`s and pass the method to its output.

  def method_missing(method_name, *args, &block)
    if Jobba::Statuses.instance_methods.include?(method_name)
      run.send(method_name, *args, &block)
    else
      super
    end
  end

  def respond_to?(method_name)
    Jobba::Statuses.instance_methods.include?(method_name) || super
  end

  def run
    _run(GET_STATUSES)
  end

  protected

  attr_accessor :clauses

  def initialize
    @clauses = []
  end

  class RunBlocks
    attr_reader :redis_block, :output_block

    def initialize(redis_block, output_block)
      @redis_block = redis_block
      @output_block = output_block
    end

    def output_block_result_is_redis_block_result?
      output_block.nil?
    end
  end

  GET_STATUSES = RunBlocks.new(
    ->(working_set, redis) {
      redis.zrange(working_set, 0, -1)
    },
    ->(ids) {
      Jobba::Statuses.new(ids)
    }
  )

  COUNT_STATUSES = RunBlocks.new(
    ->(working_set, redis) {
      redis.zcard(working_set)
    },
    nil
  )

  def _run(run_blocks)
    # Each clause in a query is converted to a sorted set (which may be filtered,
    # e.g. in the case of timestamp clauses) and then the sets are successively
    # intersected.
    #
    # Different users of this method have different uses for the final "working"
    # set.  Because we want to bundle all of the creations and intersections of
    # clause sets into one call to Redis (via a `multi` block), we have users
    # of this method provide a final block to run on the working set within
    # Redis (and within the `multi` call) and then another block to run on
    # the output of the first block.

    multi_result = redis.multi do
      load_default_clause if clauses.empty?
      working_set = nil

      clauses.each_with_index do |clause, ii|
        clause_set = clause.to_new_set

        if working_set.nil?
          working_set = clause_set
        else
          redis.zinterstore(working_set, [working_set, clause_set], weights: [0, 0])
          redis.del(clause_set)
        end
      end

      # This is later accessed as `multi_result[-2]` since it is the second to last output
      run_blocks.redis_block.call(working_set, redis)

      redis.del(working_set)
    end

    redis_block_output = multi_result[-2]

    run_blocks.output_block_result_is_redis_block_result? ?
      redis_block_output :
      run_blocks.output_block.call(redis_block_output)
  end

  def load_default_clause
    where(state: Jobba::State::ALL.collect(&:name))
  end

end
{'fraction_non_alphanumeric': 0.08052573121066271, 'fraction_numerical': 0.0022213994816734544, 'mean_word_length': 2.9698750918442323, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2084597', 'n_tokens_mistral': 1835, 'n_tokens_neox': 1769, 'n_words': 532}