<commit_before>namespace :s3 do
  desc "Upload clean assets to S3 (percentage: fraction of assets to upload; default: 0)"
  task :upload_all_clean_assets, [:percentage] => [:environment] do |_t, args|
    random_number_generator = Random.new
    Asset.where(state: 'clean').each do |asset|
      if asset.file.file.exists?
        if random_number_generator.rand(100) < args[:percentage].to_i
          asset.delay(priority: 10).save_to_cloud_storage
        end
      else
        puts "Ignoring asset (#{asset.id}) with missing file: #{asset.file.file.path}"
      end
    end
  end
end
<commit_msg>Display no. of jobs enqueued by s3:upload_all_clean_assets<commit_after>namespace :s3 do
  desc "Upload clean assets to S3 (percentage: fraction of assets to upload; default: 0)"
  task :upload_all_clean_assets, [:percentage] => [:environment] do |_t, args|
    count = 0
    random_number_generator = Random.new
    Asset.where(state: 'clean').each do |asset|
      if asset.file.file.exists?
        if random_number_generator.rand(100) < args[:percentage].to_i
          asset.delay(priority: 10).save_to_cloud_storage
          count += 1
        end
      else
        puts "Ignoring asset (#{asset.id}) with missing file: #{asset.file.file.path}"
      end
    end
    puts "Total jobs enqueued: #{count}"
  end
end

{'fraction_non_alphanumeric': 0.10959939531368103, 'fraction_numerical': 0.01436130007558579, 'mean_word_length': 3.230031948881789, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3577301', 'n_tokens_mistral': 430, 'n_tokens_neox': 418, 'n_words': 117}