<commit_before>require 'asset_processor'

namespace :govuk_assets do
  desc 'Delete file from NFS for assets uploaded to S3'
  task delete_file_from_nfs_for_assets_uploaded_to_s3: :environment do
    processor = AssetProcessor.new(scope: Asset.where(state: 'uploaded'))
    processor.process_all_assets_with do |asset_id|
      DeleteAssetFileFromNfsWorker.perform_async(asset_id)
    end
  end

  desc 'Normalize blank Asset#redirect_url values'
  task normalize_blank_asset_redirect_url_values: :environment do
    scope = Asset.where(redirect_url: '')
    result = scope.update_all('$unset' => { redirect_url: true })
    status = result.successful? ? 'OK' : 'Error'
    puts "#{status}: #{result.written_count} documents updated"
  end
end<commit_msg>Add a rake task to create a whitehall asset<commit_after>require 'asset_processor'

namespace :govuk_assets do
  desc 'Delete file from NFS for assets uploaded to S3'
  task delete_file_from_nfs_for_assets_uploaded_to_s3: :environment do
    processor = AssetProcessor.new(scope: Asset.where(state: 'uploaded'))
    processor.process_all_assets_with do |asset_id|
      DeleteAssetFileFromNfsWorker.perform_async(asset_id)
    end
  end

  desc 'Normalize blank Asset#redirect_url values'
  task normalize_blank_asset_redirect_url_values: :environment do
    scope = Asset.where(redirect_url: '')
    result = scope.update_all('$unset' => { redirect_url: true })
    status = result.successful? ? 'OK' : 'Error'
    puts "#{status}: #{result.written_count} documents updated"
  end

  desc 'Create a whitehall asset with the given legacy URL path'
  task :create_whitehall_asset, %i[file_path legacy_url_path] => :environment do |_, args|
    file_path = args[:file_path]
    legacy_url_path = args[:legacy_url_path]
    WhitehallAsset.create!(
      file: Pathname.new(file_path).open,
      legacy_url_path: legacy_url_path,
    )
  end
end
{'fraction_non_alphanumeric': 0.0911966262519768, 'fraction_numerical': 0.0021085925144965737, 'mean_word_length': 4.93125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '212985', 'n_tokens_mistral': 611, 'n_tokens_neox': 601, 'n_words': 145}