<commit_before>require "solr_wrapper"
require "elasticsearch_wrapper"
require "null_backend"
require "active_support/core_ext/module/delegation"


class Backends
  delegate :[], to: :backends

  def initialize(settings, logger = nil)
    @settings = settings
    @logger = logger || Logger.new("/dev/null")
  end

  def backends
    @backends ||= Hash.new do |hash, key|
      @logger.info "Instantiating #{key} search backend"
      backend_settings = @settings.backends[key] && @settings.backends[key].symbolize_keys
      hash[key] = backend_settings && build_backend(backend_settings)
    end
  end

private

  def build_backend(backend_settings)
    case backend_settings[:type]
    when "none"
      @logger.info "Using null backend"
      NullBackend.new(@logger)
    when "solr"
      @logger.info "Using Solr backend"
      SolrWrapper.new(
        DelSolr::Client.new(backend_settings),
        @settings.recommended_format,
        @logger,
        backend_settings[:format_filter]
      )
    when "elasticsearch"
      @logger.info "Using elasticsearch backend"
      ElasticsearchWrapper.new(
        backend_settings,
        @settings.recommended_format,
        @logger,
        backend_settings[:format_filter]
      )
    else
      raise RuntimeError, "Unknown backend '#{backend_settings[:type].inspect}'"
    end
  end
end
<commit_msg>Tweak logging levels for backends.<commit_after>require "solr_wrapper"
require "elasticsearch_wrapper"
require "null_backend"
require "active_support/core_ext/module/delegation"


class Backends
  delegate :[], to: :backends

  def initialize(settings, logger = nil)
    @settings = settings
    @logger = logger || Logger.new("/dev/null")
  end

  def backends
    @backends ||= Hash.new do |hash, key|
      @logger.debug "Instantiating #{key} search backend"
      backend_settings = @settings.backends[key] && @settings.backends[key].symbolize_keys
      hash[key] = backend_settings && build_backend(backend_settings)
    end
  end

private

  def build_backend(backend_settings)
    case backend_settings[:type]
    when "none"
      @logger.debug "Using null backend"
      NullBackend.new(@logger)
    when "solr"
      @logger.debug "Using Solr backend"
      SolrWrapper.new(
        DelSolr::Client.new(backend_settings),
        @settings.recommended_format,
        @logger,
        backend_settings[:format_filter]
      )
    when "elasticsearch"
      @logger.debug "Using elasticsearch backend"
      ElasticsearchWrapper.new(
        backend_settings,
        @settings.recommended_format,
        @logger,
        backend_settings[:format_filter]
      )
    else
      raise RuntimeError, "Unknown backend '#{backend_settings[:type].inspect}'"
    end
  end
end

{'fraction_non_alphanumeric': 0.10186199342825848, 'fraction_numerical': 0.0, 'mean_word_length': 3.308176100628931, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1715236', 'n_tokens_mistral': 813, 'n_tokens_neox': 808, 'n_words': 176}