<commit_before>class QueryBuilder
  class << self
    @@_TERM_FILTERS = [:type, :subject, :creator]
    @@_RANGE_FILTERS = [:earliest, :latest]

    def TERM_FILTERS
      @@_TERM_FILTERS
    end

    def RANGE_FILTERS
      @@_RANGE_FILTERS
    end
  end

  def initialize(opts)
    @query = opts.fetch(:query)
    @size = opts.fetch(:size)
    @from = opts.fetch(:from)
    @facets = opts.fetch(:facets, [])
    @term_filters = opts.fetch(:term_filters, {})
    @range_filters = opts.fetch(:range_filters, {})
    @sort = opts.fetch(:sort, :identifier)
    @sort_order = opts.fetch(:sort_order, :asc)
    @images = opts.fetch(:images, false)
  end

  def query
    query = {
      query: { bool: { should: [
        { query_string: { query: @query, default_operator: 'AND', default_field: 'type',        boost: 8 } },
        { query_string: { query: @query, default_operator: 'AND', default_field: 'subject',     boost: 8 } },
        { query_string: { query: @query, default_operator: 'AND', default_field: 'creator',     boost: 4 } },
        { query_string: { query: @query, default_operator: 'AND', default_field: 'title',       boost: 2 } },
        { query_string: { query: @query, default_operator: 'AND', default_field: 'description', boost: 1 } },
        { query_string: { query: @query, default_operator: 'AND', default_field: '_all',        boost: 0.5 } }
      ] } },
      from: @from,
      size: @size,
      sort: { @sort => { order: @sort_order }}
    }
    query[:facets] = facets if facets
    query[:filter] = filter if filter
    query
  end

  private

  def facets
    facets = @facets & QueryBuilder.TERM_FILTERS
    return unless facets.present?

    facets.reduce({}) do |memo, filter|
      memo[filter] = { terms: { field: filter_map(filter), size: 100 } }
      memo
    end
  end

  def filter_map(filter)
    f = {
      type: 'type.exact',
      creator: 'creator.exact'
    }[filter]
    f ||= filter.to_s
    f
  end

  def filter
    return unless (term_filters || range_filters || image_filter)

    must = []
    must.concat(term_filters) if term_filters
    must.concat(range_filters) if range_filters
    must.concat(image_filter) if image_filter
    { bool: { must: must } }
  end

  def term_filters
    return unless @term_filters.present?

    terms = @term_filters.reduce({}) do |memo, (field, value)|
      memo[field] = value.split(',').map(&:strip)
      memo
    end
    [ { terms: terms } ]
  end

  def range_filters
    return unless @range_filters.present?

    earliest = @range_filters[:earliest]
    latest = @range_filters[:latest]

    ranges = []
    ranges << { range: { latest: { gte: earliest } } } if earliest.present?
    ranges << { range: { earliest: { lte: latest } } } if latest.present?
    ranges
  end

  def image_filter
    return unless @images
    [ { exists: { field: :images } }]
  end
end<commit_msg>BUGFIX: Filtering by creator, type etc. now works as expected<commit_after>class QueryBuilder
  class << self
    @@_TERM_FILTERS = [:type, :subject, :creator]
    @@_RANGE_FILTERS = [:earliest, :latest]

    def TERM_FILTERS
      @@_TERM_FILTERS
    end

    def RANGE_FILTERS
      @@_RANGE_FILTERS
    end
  end

  def initialize(opts)
    @query = opts.fetch(:query)
    @size = opts.fetch(:size)
    @from = opts.fetch(:from)
    @facets = opts.fetch(:facets, [])
    @term_filters = opts.fetch(:term_filters, {})
    @range_filters = opts.fetch(:range_filters, {})
    @sort = opts.fetch(:sort, :identifier)
    @sort_order = opts.fetch(:sort_order, :asc)
    @images = opts.fetch(:images, false)
  end

  def query
    query = {
      query: { bool: { should: [
        { query_string: { query: @query, default_operator: 'AND', default_field: 'type',        boost: 8 } },
        { query_string: { query: @query, default_operator: 'AND', default_field: 'subject',     boost: 8 } },
        { query_string: { query: @query, default_operator: 'AND', default_field: 'creator',     boost: 4 } },
        { query_string: { query: @query, default_operator: 'AND', default_field: 'title',       boost: 2 } },
        { query_string: { query: @query, default_operator: 'AND', default_field: 'description', boost: 1 } },
        { query_string: { query: @query, default_operator: 'AND', default_field: '_all',        boost: 0.5 } }
      ] } },
      from: @from,
      size: @size,
      sort: { @sort => { order: @sort_order }}
    }
    query[:facets] = facets if facets
    query[:filter] = filter if filter
    query
  end

  private

  def facets
    facets = @facets & QueryBuilder.TERM_FILTERS
    return unless facets.present?

    facets.reduce({}) do |memo, field|
      facet = { terms: { field: field_map(field), size: 100 } }
      facet[:facet_filter] = image_filter.first if image_filter # the image filter is special as it also applies to aggregate results
      memo[field] = facet
      memo
    end
  end

  def field_map(filter)
    f = {
      type: 'type.exact',
      creator: 'creator.exact'
    }[filter]
    f ||= filter.to_s
    f
  end

  def filter
    return unless (term_filters || range_filters || image_filter)

    must = []
    must.concat(term_filters) if term_filters
    must.concat(range_filters) if range_filters
    must.concat(image_filter) if image_filter
    { bool: { must: must } }
  end

  def term_filters
    return unless @term_filters.present?

    terms = @term_filters.reduce({}) do |memo, (field, value)|
      field = field_map(field)
      memo[field] = value.split(',').map(&:strip)
      memo
    end
    [ { terms: terms } ]
  end

  def range_filters
    return unless @range_filters.present?

    earliest = @range_filters[:earliest]
    latest = @range_filters[:latest]

    ranges = []
    ranges << { range: { latest: { gte: earliest } } } if earliest.present?
    ranges << { range: { earliest: { lte: latest } } } if latest.present?
    ranges
  end

  def image_filter
    return unless @images
    [ { exists: { field: :images } } ]
  end
end
{'fraction_non_alphanumeric': 0.1357035553329995, 'fraction_numerical': 0.0033383408446002335, 'mean_word_length': 2.8385650224215246, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5777304', 'n_tokens_mistral': 2011, 'n_tokens_neox': 1962, 'n_words': 490}