<commit_before>require 'progress_bar'
require 'base64'

rows = Child.database.all_docs['rows']
return unless rows.count > 0
progressbar = ProgressBar.new rows.count
errors = []

rows.each do |row|
  begin
    doc = Child.database.get row['id']
    attachments = doc['_attachments']
    attachments.each do |attachment_id, attachment_meta|
      if attachment_meta['content_type'].start_with? 'image/'
        data = Child.database.fetch_attachment doc, attachment_id
        begin
          MiniMagick::Image.read data
        rescue MiniMagick::Invalid
          data64 = Base64.decode64 data
          MiniMagick::Image.read data64
          Child.database.put_attachment doc, attachment_id, data64
          doc = Child.database.get row['id']
        end
      end
    end if doc['_attachments']
  rescue => e
    if errors.count == 0
      puts
      puts "Logging the first error, further errors will be suppressed"
      puts e.message
      puts e.backtrace
    end

    errors << row['id']
  end

  progressbar.increment!
end

if errors.count > 0
  puts
  puts "Failed to migrate images for following records:"
  puts errors
  raise "Migration failed" unless ENV['IGNORE_0012'] == 'true'
end<commit_msg>Sanity check in image migration rapidftr/tracker#106<commit_after>rows = Child.database.all_docs['rows']
if rows.count > 0
  require 'progress_bar'
  require 'base64'

  progressbar = ProgressBar.new rows.count
  errors = []

  rows.each do |row|
    begin
      doc = Child.database.get row['id']
      attachments = doc['_attachments']
      attachments.each do |attachment_id, attachment_meta|
        if attachment_meta['content_type'].start_with? 'image/'
          data = Child.database.fetch_attachment doc, attachment_id
          begin
            MiniMagick::Image.read data
          rescue MiniMagick::Invalid
            data64 = Base64.decode64 data
            MiniMagick::Image.read data64
            Child.database.put_attachment doc, attachment_id, data64
            doc = Child.database.get row['id']
          end
        end
      end if doc['_attachments']
    rescue => e
      if errors.count == 0
        puts
        puts "Logging the first error, further errors will be suppressed"
        puts e.message
        puts e.backtrace
      end

      errors << row['id']
    end

    progressbar.increment!
  end

  if errors.count > 0
    puts
    puts "Failed to migrate images for following records:"
    puts errors
    raise "Migration failed" unless ENV['IGNORE_0012'] == 'true'
  end
end
{'fraction_non_alphanumeric': 0.08932806324110672, 'fraction_numerical': 0.016205533596837945, 'mean_word_length': 2.6105563480741796, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7379211', 'n_tokens_mistral': 759, 'n_tokens_neox': 743, 'n_words': 210}