<commit_before>require 'digest/sha1'

namespace :search do
  desc 're-index all items'
  task index: [:environment] do
    Item.find_in_batches batch_size: 500 do |items|
      Item.index.import items
    end
  end

  desc 'stage a reindex of all items'
  task stage: [:environment] do
    Tire.index(items_index_name) do
      create mappings: Item.tire.mapping
      import_all_items self
      Tire.index('items-staging').delete
      add_alias 'items-staging'
      puts "finshed generating staging index #{name}"
    end
  end

  desc 'commit the staged index to be the new index'
  task :commit do
    Tire.index('items').remove_alias('items')
    Tire.index 'items-staging' do
      add_alias 'items'
      remove_alias 'items-staging'
    end 
    puts "promoted staging to items"
  end


  def items_index_name
    "items-rebuild-#{Digest::SHA1.hexdigest(Time.now.to_s)[0,8]}"
  end

  def set_up_progress
    print '|' + ' ' * 100 + '|' + '     '
    $stdout.flush
  end

  def progress(amount)
    print "\b" * 107
    print '|' + '#' * amount + ' ' * (100 - amount) + '| '
    print ' ' if amount < 10
    print ' ' if amount < 100
    print amount
    print '%'
    $stdout.flush
  end

  def import_all_items(index)
    count = Item.count
    done = 0
    set_up_progress
    ActiveRecord::Base.logger = Logger.new
    Item.includes(:collection, :hosts, :creators, :interviewers, :interviewees, :producers, :geolocation, :contributors, :confirmed_entities, :low_scoring_entities, :middle_scoring_entities, :high_scoring_entities).includes(audio_files: :transcripts).find_in_batches batch_size: 10 do |items|
      Item.index.import items
      done += items.size
      progress done * 100 / count
    end
    puts
  end
end<commit_msg>/dev/null<commit_after>require 'digest/sha1'

namespace :search do
  desc 're-index all items'
  task index: [:environment] do
    Item.find_in_batches batch_size: 500 do |items|
      Item.index.import items
    end
  end

  desc 'stage a reindex of all items'
  task stage: [:environment] do
    Tire.index(items_index_name) do
      create mappings: Item.tire.mapping
      import_all_items self
      Tire.index('items-staging').delete
      add_alias 'items-staging'
      puts "finshed generating staging index #{name}"
    end
  end

  desc 'commit the staged index to be the new index'
  task :commit do
    Tire.index('items').remove_alias('items')
    Tire.index 'items-staging' do
      add_alias 'items'
      remove_alias 'items-staging'
    end 
    puts "promoted staging to items"
  end


  def items_index_name
    "items-rebuild-#{Digest::SHA1.hexdigest(Time.now.to_s)[0,8]}"
  end

  def set_up_progress
    print '|' + ' ' * 100 + '|' + '     '
    $stdout.flush
  end

  def progress(amount)
    print "\b" * 107
    print '|' + '#' * amount + ' ' * (100 - amount) + '| '
    print ' ' if amount < 10
    print ' ' if amount < 100
    print amount
    print '%'
    $stdout.flush
  end

  def import_all_items(index)
    count = Item.count
    done = 0
    set_up_progress
    ActiveRecord::Base.logger = Logger.new('/dev/null')
    Item.includes(:collection, :hosts, :creators, :interviewers, :interviewees, :producers, :geolocation, :contributors, :confirmed_entities, :low_scoring_entities, :middle_scoring_entities, :high_scoring_entities).includes(audio_files: :transcripts).find_in_batches batch_size: 10 do |items|
      Item.index.import items
      done += items.size
      progress done * 100 / count
    end
    puts
  end
end
{'fraction_non_alphanumeric': 0.11177644710578842, 'fraction_numerical': 0.015397775876817793, 'mean_word_length': 3.0275545350172215, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1164671', 'n_tokens_mistral': 1246, 'n_tokens_neox': 1177, 'n_words': 297}