<commit_before>module ElasticRecord
  class Task
    def self.get_models
      if class_name = ENV['CLASS']
        [class_name.camelize.constantize]
      else
        ElasticRecord::Config.models
      end
    end
  end
end

namespace :index do
  desc "Create index for CLASS or all models."
  task create: :environment do
    ElasticRecord::Task.get_models.each do |model|
      # begin
        index_name = model.elastic_index.create_and_deploy
        logger.info "Created #{model.name} index (#{index_name})"
      # rescue => e
      #   if e.message =~ /IndexAlreadyExistsException/
      #     logger.info "#{model.name} index already exists"
      #   else
      #     raise e
      #   end
      # end
    end
  end

  desc "Drop index for CLASS or all models."
  task drop: :environment do
    ElasticRecord::Task.get_models.each do |model|
      # begin
        model.elastic_index.delete_all
        logger.info "Dropped #{model.name} index"
      # rescue => e
      #   if e.message =~ /IndexMissingException/
      #     logger.info "#{model.name} index does not exist"
      #   else
      #     raise e
      #   end
      # end
    end
  end

  desc "Recreate index for CLASS or all models."
  task reset: ['index:drop', 'index:create']

  desc "Add records to index. Deploys a new index by default, or specify INDEX"
  task build: :environment do
    ElasticRecord::Task.get_models.each do |model|
      logger.info "Building #{model.name} index."

      if ENV['INDEX']
        index_name = ENV['INDEX']
      else
        logger.info "  Creating index..."
        index_name = model.elastic_index.create
      end

      logger.info "  Reindexing into #{index_name}"
      model.find_in_batches do |records|
        model.elastic_index.bulk_add(records, index_name)
      end

      logger.info "  Deploying index..."
      model.elastic_index.deploy(index_name)
<commit_msg>drop batch_size to 100 because cassandra has_many problems<commit_after>module ElasticRecord
  class Task
    def self.get_models
      if class_name = ENV['CLASS']
        [class_name.camelize.constantize]
      else
        ElasticRecord::Config.models
      end
    end
  end
end

namespace :index do
  desc "Create index for CLASS or all models."
  task create: :environment do
    ElasticRecord::Task.get_models.each do |model|
      # begin
        index_name = model.elastic_index.create_and_deploy
        logger.info "Created #{model.name} index (#{index_name})"
      # rescue => e
      #   if e.message =~ /IndexAlreadyExistsException/
      #     logger.info "#{model.name} index already exists"
      #   else
      #     raise e
      #   end
      # end
    end
  end

  desc "Drop index for CLASS or all models."
  task drop: :environment do
    ElasticRecord::Task.get_models.each do |model|
      # begin
        model.elastic_index.delete_all
        logger.info "Dropped #{model.name} index"
      # rescue => e
      #   if e.message =~ /IndexMissingException/
      #     logger.info "#{model.name} index does not exist"
      #   else
      #     raise e
      #   end
      # end
    end
  end

  desc "Recreate index for CLASS or all models."
  task reset: ['index:drop', 'index:create']

  desc "Add records to index. Deploys a new index by default, or specify INDEX"
  task build: :environment do
    ElasticRecord::Task.get_models.each do |model|
      logger.info "Building #{model.name} index."

      if ENV['INDEX']
        index_name = ENV['INDEX']
      else
        logger.info "  Creating index..."
        index_name = model.elastic_index.create
      end

      logger.info "  Reindexing into #{index_name}"
      model.find_in_batches(batch_size: 100) do |records|
        model.elastic_index.bulk_add(records, index_name)
      end

      logger.info "  Deploying index..."
      model.elastic_index.deploy(index_name)

{'fraction_non_alphanumeric': 0.0951270088128564, 'fraction_numerical': 0.0015552099533437014, 'mean_word_length': 2.3238587424633934, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5903787', 'n_tokens_mistral': 1209, 'n_tokens_neox': 1163, 'n_words': 328}