<commit_before>class OwnReality::Query

  def get(type, id)
    response = elastic.request "get", "#{type}/#{id}"
    elastic.handle resposne
  end

  def mget(type, ids)
    ids = [ids] unless ids.is_a?(Array)
    data = {
        'docs' => ids.map{|i|
          {'_index' => config['index'], '_type' => type, '_id' => i}
        }
    }
    response = elastic.request 'get', "_mget", {}, data
  end

  def elastic
    @elastic ||= OwnReality::Elastic.new
  end

  def config
    @config ||= elastic.request("get", "config/complete").last['_source']
  end

  # def ensure_scripts
  #   response = elastic.request 'post', '/_scripts/groovy/keys-for-object', {}, {
  #     'script' => '_source.attrs.by_category.keySet()'
  #   }
  #   elastic.handle(response)
  # end

  def paper(type, id)
    unless ["sources", "magazines", "articles", "interviews"].include?(type)
      raise "unknown type #{type.inspect}"
    end

    response = elastic.request "get", "#{type}/#{id}"
    elastic.handle response
  end

  def papers(type = nil, criteria = {})
    unless ["magazines", "articles", "interviews"].include?(type)
      raise "unknown type #{type.inspect}"
    end

    criteria ||= {}
    criteria["page"] = (criteria["page"] || 1).to_i
    criteria["per_page"] = (criteria["per_page"] || 10).to_i

    response = elastic.request "post", "#{type}/_search", nil, {
      "size" => criteria["per_page"],
      "from" => (criteria["page"] - 1) * criteria["per_page"],
      "sort" => ["title.de"]
    }

    elastic.handle(response)
  end

  def people(criteria = {})
    criteria['terms'] ||= ''

    data = {
      'query' => {
        'query_string' => {
          'query' => criteria['terms'],
          'default_operator' => 'AND',
          'analyzer' => 'folding',
          'analyze_wildcard' => true,
          'fields' => [
            # 'id^50',
            'last_name^10',
            'first_name^10'
          ]
        }
      }
    }

    Rails.logger.debug data.inspect

    response = elastic.request "post", "people/_search", nil, data
    elastic.handle(response)
  end

  def search(type, criteria = {})
    criteria ||= {}
    criteria["page"] = (criteria["page"] || 1).to_i
    criteria["per_page"] = (criteria["per_page"] || 10).to_i
    criteria['locale'] ||= 'de'

    search_type = criteria['search_type'] || 'query_then_fetch'
    conditions = []

    aggs = {}

    if criteria['register']
      if type == 'people'
        aggs['register'] = {
          'terms' => {
            'field' => 'initial',
            'size' => 0
          }
        }
      else
        aggs['register'] = {
          'terms' => {
            'field' => "initials.#{criteria['locale']}",
            'size' => 0
          }
        }
      end
    end

    if criteria['year_ranges']
      aggs['year_ranges'] = {
        "date_range" => {
          "field" => "from_date",
          "format" => "YYYY-MM-dd",
          "ranges" => (1960..1989).map{|i|
            {"from" => "#{i}-01-01", "to" => "#{i}-12-31"}
          }
        }
      }

      if criteria['year_ranges'] != 'true'
        conditions << {
          "range" => {
            "from_date" => {
              "lte" => Time.mktime(criteria["year_ranges"], 12, 31).strftime("%Y-%m-%dT%H:%M:%S"),
              "gte" => Time.mktime(criteria["year_ranges"], 1, 1).strftime("%Y-%m-%dT%H:%M:%S")
            }
          }
        }
      end
    end

    config["categories"].each_with_index do |data, id|
      aggs["attribs.#{id}"] = {
        "terms" => {
          # "script" => "doc['refs']['#{c}'].values",
          "field" => "attrs.by_category.#{id}",
          # "lang" => "groovy",
          "size" => 21
        }
      }

      if criteria["refs"]
        aggs["attribs.#{id}"]["terms"]["exclude"] = criteria["refs"]
      end
    end

    config['roles'].each do |id, names|
      aggs["people.#{id}"] = {
        'terms' => {
          'field' => "people.#{id}.id",
          'size' => 21
        }
      }

      if criteria["people"] && criteria['people'][id.to_s]
        aggs["people.#{id}"]["terms"]["exclude"] = criteria["people"][id.to_s]
      end
    end

    aggs['journals'] = {
      'terms' => {
        'field' => 'journal',
        'size' => 21
      }
    }

    if criteria['journals']
      aggs['journals']['terms']['exclude'] = criteria['journals']
    end

    unless type.present?
      aggs['type'] = {
        'terms' => {
          'field' => '_type',
          'size' => 10
        }
      }
    end

    data = {
      "aggs" => aggs,
      "query" => {
        "bool" => {
          "must" => conditions
        }
      },
      "size" => (search_type == 'count' ? 0 : criteria["per_page"]),
      "from" => (criteria["page"] - 1) * criteria["per_page"],
      "sort" => criteria['sort'] || {"date_from" => {'order' => 'asc', 'ignore_unmapped' => true}}
    }

    # binding.pry

    if type.present?
      data["query"]["bool"]["must"] << {
        "type" => {
          "value" => type
        }
      }
    end

    if criteria["lower"].present?
      data["query"]["bool"]["must"] << {
        "bool" => {
          "should" => [
            {
              "constant_score" => {
                "filter" => {
                  "range" => {
                    "to_date" => {
                      "gte" => Time.mktime(criteria["lower"]).strftime("%Y-%m-%dT%H:%M:%S")
                    }
                  }
                }
              }
            },{
              "constant_score" => {
                "filter" => {
                  "missing" => {
                    "field" => "to_date"
                  }
                }
              }
            }
          ]
        }
      }
    end

    if criteria["upper"].present?
      data["query"]["bool"]["must"] << {
        "bool" => {
          "should" => [
            {
              "constant_score" => {
                "filter" => {
                  "range" => {
                    "from_date" => {
                      "lte" => Time.mktime(criteria["upper"], 12, 31).strftime("%Y-%m-%dT%H:%M:%S")
                    }
                  }
                }
              }
            },{
              "constant_score" => {
                "filter" => {
                  "missing" => {
                    "field" => "from_date"
                  }
                }
              }
            }
          ]
        }
      }
    end

    if criteria["terms"].present?
      data["query"]["bool"]["must"] << {
        "query_string" => {
          "query" => criteria["terms"],
          "default_operator" => "AND",
          "analyzer" => "folding",
          "analyze_wildcard" => true,
          "fields" => [
            "title.de^20",
            "title.fr^20",
            "title.en^20",
            '_all'<commit_msg>adds moderate boost to sources's interpretation<commit_after>class OwnReality::Query

  def get(type, id)
    response = elastic.request "get", "#{type}/#{id}"
    elastic.handle resposne
  end

  def mget(type, ids)
    ids = [ids] unless ids.is_a?(Array)
    data = {
        'docs' => ids.map{|i|
          {'_index' => config['index'], '_type' => type, '_id' => i}
        }
    }
    response = elastic.request 'get', "_mget", {}, data
  end

  def elastic
    @elastic ||= OwnReality::Elastic.new
  end

  def config
    @config ||= elastic.request("get", "config/complete").last['_source']
  end

  # def ensure_scripts
  #   response = elastic.request 'post', '/_scripts/groovy/keys-for-object', {}, {
  #     'script' => '_source.attrs.by_category.keySet()'
  #   }
  #   elastic.handle(response)
  # end

  def paper(type, id)
    unless ["sources", "magazines", "articles", "interviews"].include?(type)
      raise "unknown type #{type.inspect}"
    end

    response = elastic.request "get", "#{type}/#{id}"
    elastic.handle response
  end

  def papers(type = nil, criteria = {})
    unless ["magazines", "articles", "interviews"].include?(type)
      raise "unknown type #{type.inspect}"
    end

    criteria ||= {}
    criteria["page"] = (criteria["page"] || 1).to_i
    criteria["per_page"] = (criteria["per_page"] || 10).to_i

    response = elastic.request "post", "#{type}/_search", nil, {
      "size" => criteria["per_page"],
      "from" => (criteria["page"] - 1) * criteria["per_page"],
      "sort" => ["title.de"]
    }

    elastic.handle(response)
  end

  def people(criteria = {})
    criteria['terms'] ||= ''

    data = {
      'query' => {
        'query_string' => {
          'query' => criteria['terms'],
          'default_operator' => 'AND',
          'analyzer' => 'folding',
          'analyze_wildcard' => true,
          'fields' => [
            # 'id^50',
            'last_name^10',
            'first_name^10'
          ]
        }
      }
    }

    Rails.logger.debug data.inspect

    response = elastic.request "post", "people/_search", nil, data
    elastic.handle(response)
  end

  def search(type, criteria = {})
    criteria ||= {}
    criteria["page"] = (criteria["page"] || 1).to_i
    criteria["per_page"] = (criteria["per_page"] || 10).to_i
    criteria['locale'] ||= 'de'

    search_type = criteria['search_type'] || 'query_then_fetch'
    conditions = []

    aggs = {}

    if criteria['register']
      if type == 'people'
        aggs['register'] = {
          'terms' => {
            'field' => 'initial',
            'size' => 0
          }
        }
      else
        aggs['register'] = {
          'terms' => {
            'field' => "initials.#{criteria['locale']}",
            'size' => 0
          }
        }
      end
    end

    if criteria['year_ranges']
      aggs['year_ranges'] = {
        "date_range" => {
          "field" => "from_date",
          "format" => "YYYY-MM-dd",
          "ranges" => (1960..1989).map{|i|
            {"from" => "#{i}-01-01", "to" => "#{i}-12-31"}
          }
        }
      }

      if criteria['year_ranges'] != 'true'
        conditions << {
          "range" => {
            "from_date" => {
              "lte" => Time.mktime(criteria["year_ranges"], 12, 31).strftime("%Y-%m-%dT%H:%M:%S"),
              "gte" => Time.mktime(criteria["year_ranges"], 1, 1).strftime("%Y-%m-%dT%H:%M:%S")
            }
          }
        }
      end
    end

    config["categories"].each_with_index do |data, id|
      aggs["attribs.#{id}"] = {
        "terms" => {
          # "script" => "doc['refs']['#{c}'].values",
          "field" => "attrs.by_category.#{id}",
          # "lang" => "groovy",
          "size" => 21
        }
      }

      if criteria["refs"]
        aggs["attribs.#{id}"]["terms"]["exclude"] = criteria["refs"]
      end
    end

    config['roles'].each do |id, names|
      aggs["people.#{id}"] = {
        'terms' => {
          'field' => "people.#{id}.id",
          'size' => 21
        }
      }

      if criteria["people"] && criteria['people'][id.to_s]
        aggs["people.#{id}"]["terms"]["exclude"] = criteria["people"][id.to_s]
      end
    end

    aggs['journals'] = {
      'terms' => {
        'field' => 'journal',
        'size' => 21
      }
    }

    if criteria['journals']
      aggs['journals']['terms']['exclude'] = criteria['journals']
    end

    unless type.present?
      aggs['type'] = {
        'terms' => {
          'field' => '_type',
          'size' => 10
        }
      }
    end

    data = {
      "aggs" => aggs,
      "query" => {
        "bool" => {
          "must" => conditions
        }
      },
      "size" => (search_type == 'count' ? 0 : criteria["per_page"]),
      "from" => (criteria["page"] - 1) * criteria["per_page"],
      "sort" => criteria['sort'] || {"date_from" => {'order' => 'asc', 'ignore_unmapped' => true}}
    }

    # binding.pry

    if type.present?
      data["query"]["bool"]["must"] << {
        "type" => {
          "value" => type
        }
      }
    end

    if criteria["lower"].present?
      data["query"]["bool"]["must"] << {
        "bool" => {
          "should" => [
            {
              "constant_score" => {
                "filter" => {
                  "range" => {
                    "to_date" => {
                      "gte" => Time.mktime(criteria["lower"]).strftime("%Y-%m-%dT%H:%M:%S")
                    }
                  }
                }
              }
            },{
              "constant_score" => {
                "filter" => {
                  "missing" => {
                    "field" => "to_date"
                  }
                }
              }
            }
          ]
        }
      }
    end

    if criteria["upper"].present?
      data["query"]["bool"]["must"] << {
        "bool" => {
          "should" => [
            {
              "constant_score" => {
                "filter" => {
                  "range" => {
                    "from_date" => {
                      "lte" => Time.mktime(criteria["upper"], 12, 31).strftime("%Y-%m-%dT%H:%M:%S")
                    }
                  }
                }
              }
            },{
              "constant_score" => {
                "filter" => {
                  "missing" => {
                    "field" => "from_date"
                  }
                }
              }
            }
          ]
        }
      }
    end

    if criteria["terms"].present?
      data["query"]["bool"]["must"] << {
        "query_string" => {
          "query" => criteria["terms"],
          "default_operator" => "AND",
          "analyzer" => "folding",
          "analyze_wildcard" => true,
          "fields" => [
            "title.de^20",
            "title.fr^20",
            "title.en^20",
            'interpretation.de^10',
            'interpretation.fr^10',
            'interpretation.en^10',
            '_all'
{'fraction_non_alphanumeric': 0.1938760702365404, 'fraction_numerical': 0.0087070091423596, 'mean_word_length': 1.542988929889299, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 173, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6477645', 'n_tokens_mistral': 4596, 'n_tokens_neox': 4389, 'n_words': 699}