<commit_before>require "sidekiq"
require "config"
require "sidekiq_json_encoding_patch"
require "failed_job_worker"

module Elasticsearch
  class BulkIndexWorker
    include Sidekiq::Worker
    sidekiq_options :retry => 5

    # Logger is defined on the class for use inthe `sidekiq_retries_exhausted`
    # block, and as an instance method for use the rest of the time
    def self.logger
      Logging.logger[self]
    end

    def logger
      self.class.logger
    end

    sidekiq_options :queue => :bulk

    def perform(index_name, document_hashes)
      noun = document_hashes.size > 1 ? "documents" : "document"
      logger.info "Indexing #{document_hashes.size} queued #{noun} into #{index_name}"

      index(index_name).bulk_index(document_hashes)
    end

    sidekiq_retries_exhausted do |msg|
      logger.warn "Job '#{msg["jid"]}' failed; forwarding to failure queue"
      FailedJobWorker.perform_async(msg)
    end

  private<commit_msg>Resolve a circular dependency in indexing worker.<commit_after>require "sidekiq"
require "sidekiq_json_encoding_patch"
require "failed_job_worker"

module Elasticsearch
  # This class requires the `config.rb` file to be loaded, since it requires
  # access to the `search_config` setting, but including it here can cause a
  # circular require dependency, from:
  #
  #   SearchConfig -> SearchServer -> IndexGroup -> Index -> DocumentQueue ->
  #   BulkIndexWorker -> SearchConfig
  class BulkIndexWorker
    include Sidekiq::Worker
    sidekiq_options :retry => 5

    # Logger is defined on the class for use inthe `sidekiq_retries_exhausted`
    # block, and as an instance method for use the rest of the time
    def self.logger
      Logging.logger[self]
    end

    def logger
      self.class.logger
    end

    sidekiq_options :queue => :bulk

    def perform(index_name, document_hashes)
      noun = document_hashes.size > 1 ? "documents" : "document"
      logger.info "Indexing #{document_hashes.size} queued #{noun} into #{index_name}"

      index(index_name).bulk_index(document_hashes)
    end

    sidekiq_retries_exhausted do |msg|
      logger.warn "Job '#{msg["jid"]}' failed; forwarding to failure queue"
      FailedJobWorker.perform_async(msg)
    end

  private
{'fraction_non_alphanumeric': 0.08199643493761141, 'fraction_numerical': 0.0017825311942959, 'mean_word_length': 3.619341563786008, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4877099', 'n_tokens_mistral': 706, 'n_tokens_neox': 685, 'n_words': 202}