<commit_before>namespace :rummager do

  desc "Lists current Rummager indices, pass [all] to show inactive indices"
  task :list_indices, :all do |_, args|
    show_all = args[:all] || false
    all_index_names.each do |name|
      index_group = search_server.index_group(name)
      active_index_name = index_group.current.real_name
      if show_all
        index_names = index_group.index_names
      else
        index_names = [active_index_name]
      end
      puts "#{name}:"
      index_names.sort.each do |index_name|
        if index_name == active_index_name
          puts "* #{index_name}"
        else
          puts "  #{index_name}"
        end
      end
      puts
    end
  end

  desc "Migrates an index group to a new index"
  task :migrate_index do
    index_names.each do |index_name|
      index_group = search_server.index_group(index_name)

      new_index = index_group.create_index
      old_index = index_group.current_real

      if old_index
        old_index.with_lock do
          new_index.populate_from old_index

          # Now bulk inserts fail if any of their operations fail (de47247),
          # and now we lock the old index to avoid any writes (87a7c60), the
          # document counts should always match if we get to this point without
          # throwing an exception, but it's a nice safeguard to have
          new_count = new_index.all_documents.size
          old_count = old_index.all_documents.size
          unless new_count == old_count
            logger.error(
              "Population miscount: new index has #{new_count} documents, " +
              "while old index has #{old_count}."
            )
            raise RuntimeError, "Population count mismatch"
          end

          # Switch aliases inside the lock so we avoid a race condition where a
          # new index exists, but the old index is available for writes
          index_group.switch_to new_index

          old_index.close
        end
      else
        index_group.switch_to new_index
      end
    end
  end

  desc "Switches an index group to a new index WITHOUT transferring the data"
  task :switch_to_empty_index do
    # Note that this task will effectively clear out the index, so shouldn't be
    # run on production without some serious consideration.
    index_names.each do |index_name|
      index_group = search_server.index_group(index_name)
      index_group.switch_to index_group.create_index
    end
  end

  desc "Migrates from an index with the actual index name to an alias"
  task :migrate_from_unaliased_index do
    # WARNING: this is potentially dangerous, and will leave the search
    # unavailable for a very short (sub-second) period of time
    #
    # TODO: remove this task once it is no longer needed

    index_names.each do |index_name|
      index_group = search_server.index_group(index_name)

      real_index_name = index_group.current.real_name
      unless real_index_name == index_name
        # This task only makes sense if we're migrating from an unaliased index
        raise "Expecting index name #{index_name.inspect}; found #{real_index_name.inspect}"
      end

      logger.info "Creating new #{index_name} index..."
      new_index = index_group.create_index
      logger.info "...index '#{new_index.real_name}' created"

      logger.info "Populating new #{index_name} index..."
      new_index.populate_from index_group.current
      logger.info "...index populated."

      logger.info "Deleting #{index_name} index..."
      index_group.send :delete, CGI.escape(index_name)
      logger.info "...deleted."

      logger.info "Switching #{index_name}..."<commit_msg>Remove old TODO from Rakefile.<commit_after>namespace :rummager do

  desc "Lists current Rummager indices, pass [all] to show inactive indices"
  task :list_indices, :all do |_, args|
    show_all = args[:all] || false
    all_index_names.each do |name|
      index_group = search_server.index_group(name)
      active_index_name = index_group.current.real_name
      if show_all
        index_names = index_group.index_names
      else
        index_names = [active_index_name]
      end
      puts "#{name}:"
      index_names.sort.each do |index_name|
        if index_name == active_index_name
          puts "* #{index_name}"
        else
          puts "  #{index_name}"
        end
      end
      puts
    end
  end

  desc "Migrates an index group to a new index"
  task :migrate_index do
    index_names.each do |index_name|
      index_group = search_server.index_group(index_name)

      new_index = index_group.create_index
      old_index = index_group.current_real

      if old_index
        old_index.with_lock do
          new_index.populate_from old_index

          # Now bulk inserts fail if any of their operations fail (de47247),
          # and now we lock the old index to avoid any writes (87a7c60), the
          # document counts should always match if we get to this point without
          # throwing an exception, but it's a nice safeguard to have
          new_count = new_index.all_documents.size
          old_count = old_index.all_documents.size
          unless new_count == old_count
            logger.error(
              "Population miscount: new index has #{new_count} documents, " +
              "while old index has #{old_count}."
            )
            raise RuntimeError, "Population count mismatch"
          end

          # Switch aliases inside the lock so we avoid a race condition where a
          # new index exists, but the old index is available for writes
          index_group.switch_to new_index

          old_index.close
        end
      else
        index_group.switch_to new_index
      end
    end
  end

  desc "Switches an index group to a new index WITHOUT transferring the data"
  task :switch_to_empty_index do
    # Note that this task will effectively clear out the index, so shouldn't be
    # run on production without some serious consideration.
    index_names.each do |index_name|
      index_group = search_server.index_group(index_name)
      index_group.switch_to index_group.create_index
    end
  end

  desc "Migrates from an index with the actual index name to an alias"
  task :migrate_from_unaliased_index do
    # WARNING: this is potentially dangerous, and will leave the search
    # unavailable for a very short (sub-second) period of time

    index_names.each do |index_name|
      index_group = search_server.index_group(index_name)

      real_index_name = index_group.current.real_name
      unless real_index_name == index_name
        # This task only makes sense if we're migrating from an unaliased index
        raise "Expecting index name #{index_name.inspect}; found #{real_index_name.inspect}"
      end

      logger.info "Creating new #{index_name} index..."
      new_index = index_group.create_index
      logger.info "...index '#{new_index.real_name}' created"

      logger.info "Populating new #{index_name} index..."
      new_index.populate_from index_group.current
      logger.info "...index populated."

      logger.info "Deleting #{index_name} index..."
      index_group.send :delete, CGI.escape(index_name)
      logger.info "...deleted."

      logger.info "Switching #{index_name}..."
{'fraction_non_alphanumeric': 0.06800660792951542, 'fraction_numerical': 0.0027533039647577094, 'mean_word_length': 2.5823471400394475, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2605468', 'n_tokens_mistral': 2106, 'n_tokens_neox': 2039, 'n_words': 675}