<commit_before>require 'solr'

class ModelSet
  class SolrQuery < Query
    include Conditioned

    MAX_SOLR_RESULTS = 1000

    def anchor!(query)
      add_conditions!( ids_clause(query.ids) )
    end    

    def size
      fetch_results if @size.nil?
      @size
    end

    def count
      fetch_results if @count.nil?
      @count
    end

    def ids
      fetch_results if @ids.nil?
      @ids
    end

    def resp
      fetch_results if @resp.nil?
      @resp
    end

  private

    def fetch_results
      query = "#{conditions.to_s}"
      solr_params = {:highlighting => {}}

      if limit
        solr_params[:rows]  = limit
        solr_params[:start] = offset
      else
        solr_params[:rows] = MAX_SOLR_RESULTS
      end

      before_query(solr_params)
      begin 
        @resp = Solr::Connection.new(SOLR_HOST).search(query, solr_params)
      rescue Exception => e
        on_exception(e, solr_params)
      end
      after_query(solr_params)

      @count = @resp.total_hits
      @ids   = @resp.hits.map{ |hit| hit["discussion_id"].to_i }
      @size  = @ids.size
    end

    def ids_clause(ids, field = nil)
      return 'pk_i:(false)' if ids.empty?
      field ||= 'pk_i'
      "#{field}:(#{ids.join(' OR ')})"
    end

    def sanitize_condition(condition)
      condition
    end
  end
end<commit_msg>clean up solr query response<commit_after>require 'solr'

class ModelSet
  class SolrQuery < Query
    attr_reader :response
    include Conditioned

    MAX_SOLR_RESULTS = 1000

    def anchor!(query)
      add_conditions!( ids_clause(query.ids) )
    end    

    def size
      fetch_results if @size.nil?
      @size
    end

    def count
      fetch_results if @count.nil?
      @count
    end

    def ids
      fetch_results if @ids.nil?
      @ids
    end

  private

    def fetch_results
      query = "#{conditions.to_s}"
      solr_params = {:highlighting => {}}

      if limit
        solr_params[:rows]  = limit
        solr_params[:start] = offset
      else
        solr_params[:rows] = MAX_SOLR_RESULTS
      end

      before_query(solr_params)
      begin 
        @response = Solr::Connection.new(SOLR_HOST).search(query, solr_params)
      rescue Exception => e
        on_exception(e, solr_params)
      end
      after_query(solr_params)

      @count = @response.total_hits
      @ids   = @response.hits.map{ |hit| hit["discussion_id"].to_i }
      @size  = @ids.size
    end

    def ids_clause(ids, field = nil)
      return 'pk_i:(false)' if ids.empty?
      field ||= 'pk_i'
      "#{field}:(#{ids.join(' OR ')})"
    end

    def sanitize_condition(condition)
      condition
    end
  end
end
{'fraction_non_alphanumeric': 0.1002628614344724, 'fraction_numerical': 0.0030041306796845663, 'mean_word_length': 2.156398104265403, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2199529', 'n_tokens_mistral': 930, 'n_tokens_neox': 882, 'n_words': 200}