<commit_before>class CleanJob < ApplicationJob
  queue_as :default

  def perform(*args)
    batch_range = 0..(Batch.active_marker-1)
    Batch.where(marker: batch_range).destroy_all
  end
end<commit_msg>Sophisticate clean_job to remove jobs after active<commit_after>class CleanJob < ApplicationJob
  queue_as :default

  def perform(*args)

    start_batch = 1
    end_batch = Batch.active_marker-1

    purge_batch_range start_batch, end_batch

    start_batch = Batch.active_marker+1
    end_batch = Batch.current_marker-1

    purge_batch_range start_batch, end_batch

  end

  private

    def purge_batch_range start_batch, end_batch
      batch_range = start_batch..end_batch
      return if batch_range.count.zero?

      Batch.where(marker: batch_range).destroy_all
    end

end
{'fraction_non_alphanumeric': 0.06598984771573604, 'fraction_numerical': 0.007614213197969543, 'mean_word_length': 3.8703703703703702, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '729546', 'n_tokens_mistral': 266, 'n_tokens_neox': 250, 'n_words': 54}