<commit_before># Looks at the Effective::Asset and Effective::Attachments and converts to ActiveStorage
#
# 1. Put in the has_attached_files to each model to be upgraded
# 2. Remove the acts_as_asset_box

require 'timeout'

module Effective
  class AssetReplacer
    include ActiveStorage::Blob::Analyzable

    BATCH_SIZE = 500
    TIMEOUT = 120

    def replace!(skip_existing: false)
      verify!

      attachments = attachments_to_process().to_a

      while(true)
        wait_for_active_job!

        puts "\nEnqueuing #{attachments.length} attachments with ids #{attachments.first.id} to #{attachments.last.id}"

        attachments.each do |attachment|
          asset = attachment.asset
          attachable = attachment.attachable
          next if asset.blank? || attachable.blank?

          box = attachment.box.singularize
          boxes = attachment.box

          one = attachable.respond_to?(box) && attachable.send(box).kind_of?(ActiveStorage::Attached::One)
          many = attachable.respond_to?(boxes) && attachable.send(boxes).kind_of?(ActiveStorage::Attached::Many)
          box = (one ? box : boxes)

          if skip_existing
            existing = Array(attachable.send(box))

            if existing.any? { |obj| obj.respond_to?(:filename) && obj.filename.to_s == asset.file_name }
              puts("Skipping existing #{attachable.class.name} #{attachable.id} #{box} #{asset.file_name}.")
              next
            end
          end

          Effective::AssetReplacerJob.perform_later(attachment, box)
        end

        attachments = attachments_to_process().where.not(id: attachments.map(&:id))
        break if attachments.to_a.blank?

        GC.start
      end

      puts "\nAll Done. Have a great day."
      true
    end

    # This is called on the background by the AssetReplacerJob
    def replace_attachment!(attachment, box)
      raise('expected an Effective::Attachment') unless attachment.kind_of?(Effective::Attachment)
      puts("Processing attachment ##{attachment.id}")

      asset = attachment.asset
      attachable = attachment.attachable

      attachable.replacing_asset = true if attachable.respond_to?(:replacing_asset=)

      Timeout.timeout(TIMEOUT) do
        attachable.send(box).attach(
          io: URI.open(asset.url),
          filename: asset.file_name,
          content_type: asset.content_type.presence,
          identify: (asset.content_type.blank?)
        )

        attachment.update_column(:replaced, true)
      end

      true<commit_msg>CGI unescape the filenames<commit_after># Looks at the Effective::Asset and Effective::Attachments and converts to ActiveStorage
#
# 1. Put in the has_attached_files to each model to be upgraded
# 2. Remove the acts_as_asset_box

require 'timeout'
require 'cgi'

module Effective
  class AssetReplacer
    include ActiveStorage::Blob::Analyzable

    BATCH_SIZE = 500
    TIMEOUT = 120

    def replace!(skip_existing: false)
      verify!

      attachments = attachments_to_process().to_a

      while(true)
        wait_for_active_job!

        puts "\nEnqueuing #{attachments.length} attachments with ids #{attachments.first.id} to #{attachments.last.id}"

        attachments.each do |attachment|
          asset = attachment.asset
          attachable = attachment.attachable
          next if asset.blank? || attachable.blank?

          box = attachment.box.singularize
          boxes = attachment.box

          one = attachable.respond_to?(box) && attachable.send(box).kind_of?(ActiveStorage::Attached::One)
          many = attachable.respond_to?(boxes) && attachable.send(boxes).kind_of?(ActiveStorage::Attached::Many)
          box = (one ? box : boxes)

          if skip_existing
            existing = Array(attachable.send(box))

            if existing.any? { |obj| obj.respond_to?(:filename) && obj.filename.to_s == asset.file_name }
              puts("Skipping existing #{attachable.class.name} #{attachable.id} #{box} #{asset.file_name}.")
              next
            end
          end

          Effective::AssetReplacerJob.perform_later(attachment, box)
        end

        attachments = attachments_to_process().where.not(id: attachments.map(&:id))
        break if attachments.to_a.blank?

        GC.start
      end

      puts "\nAll Done. Have a great day."
      true
    end

    # This is called on the background by the AssetReplacerJob
    def replace_attachment!(attachment, box)
      raise('expected an Effective::Attachment') unless attachment.kind_of?(Effective::Attachment)
      puts("Processing attachment ##{attachment.id}")

      asset = attachment.asset
      attachable = attachment.attachable

      attachable.replacing_asset = true if attachable.respond_to?(:replacing_asset=)

      Timeout.timeout(TIMEOUT) do
        attachable.send(box).attach(
          io: URI.open(asset.url),
          filename: CGI.unescape(asset.file_name.to_s),
          content_type: asset.content_type.presence,
          identify: (asset.content_type.blank?)
        )

        attachment.update_column(:replaced, true)
      end

      true
{'fraction_non_alphanumeric': 0.10019569471624266, 'fraction_numerical': 0.0031311154598825833, 'mean_word_length': 2.730656934306569, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2633792', 'n_tokens_mistral': 1500, 'n_tokens_neox': 1433, 'n_words': 328}