<commit_before>require 'ostruct'
require 'set'

class SearcherService
  include ActionView::Helpers::TextHelper

  attr_accessor :user, :q, :ids

  def initialize(user)
    @user = user
  end

  def where(str, document_id=nil)
    store(str)
    client = Rails.application.config.elasticsearch_client

    query = {
      bool: {
        must:[
          {
            query_string: { query: str }
          },
          {
            term: {
              user_id: { term: user.id}
            }
          }
        ]
      }
    }
    query[:bool][:must] << { term: {document_id: {term: document_id }}} if document_id

    puts query.inspect

    highlight = {
      fields: { text:{}}
    }

    response = client.search index: "#{documents_index},#{pages_index}", body: { from: 0, size: 50, query: query, highlight: highlight }
    process_response(response)
  end

  def process_response(response)
    output = []

    Document.find(get_document_ids(response)).each do |document|
      page_results = response.fetch('hits', {})['hits'].select do |result|
        result['_index'] == 'pages_development' && result['_source']['document_id'] == document.id.to_s
      end
      output << if page_results.empty?
        build_document_result(document)
      else
        build_document_with_pages_result(document, page_results)
      end
    end
    output
  end

  def get_document_ids(response)
    output = Set.new
    response.fetch('hits', {})['hits'].each do |result|
      output.add(result['_source']['document_id'])
    end
    output.to_a
  end

  def build_document_result(document)
    {
      title: document.title,
      original_filename: document.original_filename,
      document_id: document.id,
      created_at: document.created_at,
      counters: build_counters(document)
    }
  end<commit_msg>[aphrodite] Fix pages search<commit_after>require 'ostruct'
require 'set'

class SearcherService
  include ActionView::Helpers::TextHelper

  attr_accessor :user, :q, :ids

  def initialize(user)
    @user = user
  end

  def where(str, document_id=nil)
    store(str)
    client = Rails.application.config.elasticsearch_client

    query = {
      bool: {
        must:[
          {
            query_string: { query: str }
          },
          {
            term: {
              user_id: { term: user.id}
            }
          }
        ]
      }
    }
    query[:bool][:must] << { term: {document_id: {term: document_id }}} if document_id

    puts query.inspect

    highlight = {
      fields: { text:{}}
    }

    response = client.search index: "#{documents_index},#{pages_index}", body: { from: 0, size: 50, query: query, highlight: highlight }
    process_response(response)
  end

  def process_response(response)
    output = []

    Document.find(get_document_ids(response)).each do |document|
      page_results = response.fetch('hits', {})['hits'].select do |result|
        result['_index'] == pages_index && result['_source']['document_id'] == document.id.to_s
      end
      output << if page_results.empty?
        build_document_result(document)
      else
        build_document_with_pages_result(document, page_results)
      end
    end
    output
  end

  def get_document_ids(response)
    output = Set.new
    response.fetch('hits', {})['hits'].each do |result|
      output.add(result['_source']['document_id'])
    end
    output.to_a
  end

  def build_document_result(document)
    {
      title: document.title,
      original_filename: document.original_filename,
      document_id: document.id,
      created_at: document.created_at,
      counters: build_counters(document)
    }
  end
{'fraction_non_alphanumeric': 0.1196815811144661, 'fraction_numerical': 0.0016469942355201758, 'mean_word_length': 2.600790513833992, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1548907', 'n_tokens_mistral': 1139, 'n_tokens_neox': 1114, 'n_words': 226}