<commit_before>require_relative '../init.rb'

module HubIndexer
  class Runner
    def initialize(options, profile, &block)
      @options        = options
      @indexer        = indexer(options)
      @profile        = profile
      @push_count     = options[:solr_push_count]
      @is_test        = options[:test]
      @delete_by      = options[:delete_by]
    end

    def fetch_originals(opts)
      if opts[:bucket]
        return Bucket.new(opts)
      elsif opts[:directory]
        return LocalRecords.new(opts)
      else
        raise "Must specify either a bucket or local file directory option"
      end
    end

    def run
      upsert unless @delete_by
      @indexer.delete_by(@delete_by) if @delete_by
      @indexer.commit
    end

    def upsert
      i = 1
      records = []
      fetch_originals(@options).each do |key, original|
        @fetch_listener.call(i, key) if @fetch_listener
        original['id'] = key<commit_msg>regression: missing batch id after last big refactoring<commit_after>require_relative '../init.rb'

module HubIndexer
  class Runner
    def initialize(options, profile, &block)
      @options        = options
      @indexer        = indexer(options)
      @profile        = profile
      @push_count     = options[:solr_push_count]
      @is_test        = options[:test]
      @delete_by      = options[:delete_by]
      @batch_id       = options[:batch_id]
    end

    def fetch_originals(opts)
      if opts[:bucket]
        return Bucket.new(opts)
      elsif opts[:directory]
        return LocalRecords.new(opts)
      else
        raise "Must specify either a bucket or local file directory option"
      end
    end

    def run
      upsert unless @delete_by
      @indexer.delete_by(@delete_by) if @delete_by
      @indexer.commit
    end

    def upsert
      i = 1
      records = []
      fetch_originals(@options).each do |key, original|
        @fetch_listener.call(i, key) if @fetch_listener
        original['id'] = key
{'fraction_non_alphanumeric': 0.08748114630467571, 'fraction_numerical': 0.001005530417295123, 'mean_word_length': 2.2148626817447497, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4357505', 'n_tokens_mistral': 592, 'n_tokens_neox': 583, 'n_words': 152}