<commit_before>require 'rubygems'
require 'rsolr'
require 'elasticsearch'

module Krikri
  ##
  # Search index base class that gets extended by QA and Production index
  # classes
  #
  # @todo rewrite to use generalized `EntityConsumer` interface & avoid 
  #   `#update_from_activity`, which is tighly bound to `Activity` rather than
  #   `Enumerator<#entities>`.
  class SearchIndex
    def initialize(opts)
      @bulk_update_size = opts.delete(:bulk_update_size) { 10 }
    end

    ##
    # Add a single JSON document to the search index.
    # Implemented in a child class.
    #
    # @param _ [Hash] Hash that can be serialized to JSON with #to_json
    def add(_)
      fail NotImplementedError
    end

    ##
    # Add a number of JSON documents to the search index at once.
    # Implemented in a child class.
    #
    # @param _ [Array] Hashes that can be serialized to JSON with #to_json
    def bulk_add(_)<commit_msg>Bump bulk update size for indexing to 1,000 items<commit_after>require 'rubygems'
require 'rsolr'
require 'elasticsearch'

module Krikri
  ##
  # Search index base class that gets extended by QA and Production index
  # classes
  #
  # @todo rewrite to use generalized `EntityConsumer` interface & avoid 
  #   `#update_from_activity`, which is tighly bound to `Activity` rather than
  #   `Enumerator<#entities>`.
  class SearchIndex
    def initialize(opts)
      @bulk_update_size = opts.delete(:bulk_update_size) { 1000 }
    end

    ##
    # Add a single JSON document to the search index.
    # Implemented in a child class.
    #
    # @param _ [Hash] Hash that can be serialized to JSON with #to_json
    def add(_)
      fail NotImplementedError
    end

    ##
    # Add a number of JSON documents to the search index at once.
    # Implemented in a child class.
    #
    # @param _ [Array] Hashes that can be serialized to JSON with #to_json
    def bulk_add(_)
{'fraction_non_alphanumeric': 0.07486910994764398, 'fraction_numerical': 0.005235602094240838, 'mean_word_length': 3.0573248407643314, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5255762', 'n_tokens_mistral': 588, 'n_tokens_neox': 572, 'n_words': 221}