<commit_before>require 'action_view'
require 'rainbow'

include ActionView::Helpers::NumberHelper

namespace :premigrate do

  ALL_ROWS = 1_000_000

  def logger
    Rails.logger
  end

  def get_all_batches
    batches = []
    query = Solrizer.solr_name('has_model', :symbol) + ':"' + Batch.to_class_uri + '"'
    results = ActiveFedora::SolrService.query(query, fl: "id", rows: ALL_ROWS)
    results.each do |batch|
      batch_id = batch["id"]
      if valid_id?(batch_id) && !batches.include?(batch_id)
        batches.push batch_id 
      end
    end
    batches
  end

  # batch_token should be an array of one element
  # ["info:fedora/scholarsphere:<PASSWORD>"]
  def get_batch_id(batch_token)
    return if batch_token.blank?
    batch_id = batch_token.first.split('/').last
    batch_id
  end

  def valid_id?(identifier)
    identifier.to_s.start_with?(ScholarSphere::Application.config.id_namespace)
  end

  def get_batches_in_files
    batches = []
    query = Solrizer.solr_name('has_model', :symbol) + ':"' + GenericFile.to_class_uri + '"' 
    results = ActiveFedora::SolrService.query(query, fl: "id is_part_of_ssim", rows: ALL_ROWS)
    results.each do |file|
      begin
        batch_id = get_batch_id file['is_part_of_ssim'] 
        if valid_id?(batch_id) && !batches.include?(batch_id) 
          batches.push batch_id   
        end
      rescue Exception => e  
        logger.error "Error getting batch from file #{file}"
      end
    end
    batches
  end

  def get_missing_batches(verbose)
    all_batches = get_all_batches
    batches_in_files = get_batches_in_files
    missing_batches = []
    batches_in_files.each do |id|
      unless all_batches.include? id
        missing_batches.push(id)
      end
    end
    if verbose
      logger.info "Total batches: #{all_batches.length}"
      logger.info all_batches
      logger.info "Batches in files: #{batches_in_files.length}"
      logger.info batches_in_files
    end
    missing_batches<commit_msg>Make sure we don't create a batch with the same ID as an another existing object (e.g. we don't overwrite a GenericFile)<commit_after>require 'action_view'
require 'rainbow'

include ActionView::Helpers::NumberHelper

namespace :premigrate do

  ALL_ROWS = 1_000_000

  def logger
    Rails.logger
  end

  def get_all_batches
    batches = []
    query = Solrizer.solr_name('has_model', :symbol) + ':"' + Batch.to_class_uri + '"'
    results = ActiveFedora::SolrService.query(query, fl: "id", rows: ALL_ROWS)
    results.each do |batch|
      batch_id = batch["id"]
      if valid_id?(batch_id) && !batches.include?(batch_id)
        batches.push batch_id 
      end
    end
    batches
  end

  # batch_token should be an array of one element
  # ["info:fedora/scholarsphere:g732d900g"]
  def get_batch_id(batch_token)
    return if batch_token.blank?
    batch_id = batch_token.first.split('/').last
    batch_id
  end

  def valid_id?(identifier)
    identifier.to_s.start_with?(ScholarSphere::Application.config.id_namespace)
  end

  def get_batches_in_files
    batches = []
    query = Solrizer.solr_name('has_model', :symbol) + ':"' + GenericFile.to_class_uri + '"' 
    results = ActiveFedora::SolrService.query(query, fl: "id is_part_of_ssim", rows: ALL_ROWS)
    results.each do |file|
      begin
        batch_id = get_batch_id file['is_part_of_ssim'] 
        if valid_id?(batch_id) && !batches.include?(batch_id) 
          batches.push batch_id   
        end
      rescue Exception => e  
        logger.error "Error getting batch from file #{file}"
      end
    end
    batches
  end

  def get_missing_batches(verbose)
    all_batches = get_all_batches
    batches_in_files = get_batches_in_files
    missing_batches = []
    batches_in_files.each do |id|
      unless all_batches.include? id
        if ActiveFedora::Base.exists?(pid: id)
          # An object with this PID already exists 
          # (it might exist as something other than a Batch)
          logger.error "Skipping #{id} because it already exists!"
        else
          missing_batches.push(id)
        end
      end
    end
    if verbose
      logger.info "Total batches: #{all_batches.length}"
      logger.info all_batches
      logger.info "Batches in files: #{batches_in_files.length}"
      logger.info batches_in_files
    end
    missing_batches
{'fraction_non_alphanumeric': 0.0959409594095941, 'fraction_numerical': 0.004612546125461255, 'mean_word_length': 3.064667291471415, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4521268', 'n_tokens_mistral': 1505, 'n_tokens_neox': 1460, 'n_words': 324}