<commit_before>class SmallThumbnails < ActiveRecord::Migration
  def self.up
    total_documents = Document.all.count
    Document.all.each_with_index do |doc, i|
      begin
        puts " --> #{i+1}/#{total_documents} Starting: [#{doc.page_count} pages] #{doc.title}"
        asset_store ||= DC::Store::AssetStore.new
        pages = {}
        doc.page_count.times.each do |page|
          pages[page] = "#{page+1}-normal.gif"
          page_path = doc.page_image_path(page+1, 'normal')
          File.open(pages[page], 'w+') do |f|
            f.write(asset_store.read(page_path))
          end
        end

        puts " --> Processing #{doc.page_count} images..."
        cmd = "OMP_NUM_THREADS=2 gm mogrify -limit memory 256MiB -limit map 512MiB -density 150 -resize #{Page::IMAGE_SIZES['small']} -quality 100 -unsharp 0x0.5+0.75 \"*.gif\" 2>&1"
        result = `#{cmd}`

        puts " --> Saving #{doc.page_count} images..."
        pages.each_pair do |i, page|
          asset_store.save_page_images(doc, i + 1,
            {'small' => page},
            doc.access
          )
          File.delete(page)
        end
      rescue<commit_msg>Fixing up small thumbnails migration to not reference a size that only rexists on the addremove branch.<commit_after>class SmallThumbnails < ActiveRecord::Migration
  def self.up
    total_documents = Document.all.count
    Document.all.each_with_index do |doc, i|
      begin
        puts " --> #{i+1}/#{total_documents} Starting: [#{doc.page_count} pages] #{doc.title}"
        asset_store = DC::Store::AssetStore.new
        pages = {}
        doc.page_count.times.each do |page|
          pages[page] = "#{page+1}-normal.gif"
          page_path = doc.page_image_path(page+1, 'normal')
          File.open(pages[page], 'w+') do |f|
            f.write(asset_store.read(page_path))
          end
        end

        puts " --> Processing #{doc.page_count} images..."
        cmd = "OMP_NUM_THREADS=2 gm mogrify -limit memory 256MiB -limit map 512MiB -density 150 -resize 180x -quality 100 -unsharp 0x0.5+0.75 \"*.gif\" 2>&1"
        result = `#{cmd}`

        puts " --> Saving #{doc.page_count} images..."
        pages.each_pair do |i, page|
          asset_store.save_page_images(doc, i + 1,
            {'small' => page},
            doc.access
          )
          File.delete(page)
        end
      rescue
{'fraction_non_alphanumeric': 0.14086883171657527, 'fraction_numerical': 0.022353437368199072, 'mean_word_length': 2.2054054054054055, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6710649', 'n_tokens_mistral': 807, 'n_tokens_neox': 771, 'n_words': 173}