<commit_before>namespace :rummager do
  desc "Reindex published documents"
  task :index => ['rummager:index:detailed', 'rummager:index:government']

  namespace :index do
    task :government => :environment do
      Rummageable.index(Whitehall.government_search_index, Whitehall.government_search_index_name)
      Rummageable.commit(Whitehall.government_search_index_name)
    end

    task :detailed => :environment do
      Rummageable.index(Whitehall.detailed_guidance_search_index, Whitehall.detailed_guidance_search_index_name)
      Rummageable.commit(Whitehall.detailed_guidance_search_index_name)
    end<commit_msg>Indicate that rummager tasks do nothing in dev<commit_after>namespace :rummager do
  desc "Reindex published documents"
  task :index => ['rummager:index:detailed', 'rummager:index:government']

  namespace :index do
    task :warn_about_no_op do
      if Rails.env.development? && ENV["RUMMAGER_HOST"].blank?
        puts "Note: not actually submitting content to Rummager. Set RUMMAGER_HOST if you want to do so." 
      end
    end

    task :government => [:environment, :warn_about_no_op] do
      Rummageable.index(Whitehall.government_search_index, Whitehall.government_search_index_name)
      Rummageable.commit(Whitehall.government_search_index_name)
    end

    task :detailed => [:environment, :warn_about_no_op] do
      Rummageable.index(Whitehall.detailed_guidance_search_index, Whitehall.detailed_guidance_search_index_name)
      Rummageable.commit(Whitehall.detailed_guidance_search_index_name)
    end
{'fraction_non_alphanumeric': 0.07945736434108527, 'fraction_numerical': 0.0, 'mean_word_length': 5.196, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4787890', 'n_tokens_mistral': 499, 'n_tokens_neox': 480, 'n_words': 92}