<commit_before>require 'image_helper'
require 'open-uri' # TODO: Move elsewhere
namespace :fromthepage do

  desc "Resize image file or directories of image files"
  task :compress_images, [:pathname] => :environment  do  |t,args|
    pathname = args.pathname
    p "compressing #{pathname}"
    
    if Dir.exist? pathname
      ImageHelper.compress_files_in_dir(pathname)
    else
      # this is a single file
      ImageHelper.compress_file(pathname)
    end
  end
  IMAGE_FILE_EXTENSIONS = ['jpg', 'JPG', 'jpeg', 'JPEG', 'png', 'PNG']
  IMAGE_FILE_EXTENSIONS_PATTERN = /jpg|JPG|jpeg|JPEG|png|PNG/
  TIFF_FILE_EXTENSIONS_PATTERN = /tif|TIF|tiff|TIFF/

  desc "Process a document upload"
  task :process_document_upload, [:document_upload_id] => :environment do |t,args|
    require "#{Rails.root}/app/helpers/error_helper"
    include ErrorHelper

    document_upload_id = args.document_upload_id
    print "fetching upload with ID=#{document_upload_id}\n"
    document_upload = DocumentUpload.find document_upload_id
    
    print "found document_upload for \n\tuser=#{document_upload.user.login}, \n\ttarget collection=#{document_upload.collection.title}, \n\tfile=#{document_upload.file}\n"
    
    document_upload.status = DocumentUpload::Status::PROCESSING
    document_upload.save
    
    process_batch(document_upload, File.dirname(document_upload.file.path), document_upload.id.to_s)

    document_upload.status = DocumentUpload::Status::FINISHED
    document_upload.save

    #if the upload processes correctly,
    #remove the uploaded file to prevent filling up the disk
    if document_upload.status = DocumentUpload::Status::FINISHED
      document_upload.remove_file!
      document_upload.save
    end

    if SMTP_ENABLED
      begin
        SystemMailer.upload_succeeded(document_upload).deliver!
        UserMailer.upload_finished(document_upload).deliver!
      rescue StandardError => e
        print "SMTP Failed: Exception: #{e.message}"
      end
    end

  end


  def process_batch(document_upload, path, temp_dir_seed)
    # copy to temp dir
    temp_dir = temp_dir_path(temp_dir_seed)
    copy_to_temp_dir(path, temp_dir)

    # unzip everything
    unzip_tree(temp_dir)
    # extract any pdfs
    unpdf_tree(temp_dir, document_upload.ocr)
    #convert tiffs to jpgs
    untiff_tree(temp_dir)
    # resize files
    compress_tree(temp_dir)
    # ingest
    ingest_tree(document_upload, temp_dir)
    # clean
    clean_tmp_dir(temp_dir)
  end
  
  def clean_tmp_dir(temp_dir)
    print "Removing #{temp_dir}\n"
    FileUtils::rm_r(temp_dir)
  end
  
  def unzip_tree(temp_dir)
    print "unzip_tree(#{temp_dir})\n"
    ls = Dir.glob(File.join(temp_dir, "*"))
    ls.each do |path|
      print "\tunzip_tree considering #{path}\n"
      if Dir.exist? path
        print "Found directory #{path}\n"
        unzip_tree(path) #recurse
      else
        if File.extname(path) == '.ZIP' || File.extname(path) == '.zip'
          print "Found zipfile #{path}\n"
          #unzip and recur
          destination = File.join(File.dirname(path), File.basename(path).sub(File.extname(path),''))
          print "Calling unzip_file(#{path}, #{destination})\n"
          ImageHelper.unzip_file(path, destination)
          unzip_tree(destination)  # recurse
        end
      end
    end
    FileUtils.chmod_R "u=rwx,go=r", temp_dir
  end
  
  def unpdf_tree(temp_dir, ocr)
    print "unpdf_tree(#{temp_dir})\n"
    ls = Dir.glob(File.join(temp_dir, "*"))
    ls.each do |path|
      print "\tunpdf_tree considering #{path})\n"
      if Dir.exist? path
        print "\tunpdf_tree Found directory #{path}\n"
        unpdf_tree(path, ocr) #recurse
      else
        if File.extname(path) == '.PDF' || File.extname(path) == '.pdf'
          print "\t\tunpdf_tree Found pdf #{path}\n"
          #extract 
          destination = ImageHelper.extract_pdf(path, ocr)
          print "\t\tunpdf_tree Extracted to #{destination}\n"
          # copy any metadata.yml to the destination
          metadata_fn = File.join(File.dirname(path), 'metadata.yml')
          if File.exist? metadata_fn
            print "\t\tunpdf_tree Copy #{metadata_fn} to #{destination}\n"
            FileUtils.cp(metadata_fn, destination)
          else
            print "\t\tunpdf_tree No metadata file exists at #{metadata_fn}\n"
          end
        end
      end
    end
  end

  def untiff_tree(temp_dir)
    print "convert tiffs from tree(#{temp_dir})\n"
    ls = Dir.glob(File.join(temp_dir, "*"))
    ls.each do |path|
      print "\tuntiff_tree considering #{path})\n"
      if Dir.exist? path
        print "Found directory #{path}\n"
        untiff_tree(path) #recurse
      else
        if File.extname(path).match TIFF_FILE_EXTENSIONS_PATTERN
          print "Found tiff #{path}\n"
          #convert tiff to jpg
          destination = ImageHelper.convert_tiff(path)
          GC.start
        end
      end
    end

  end

  def compress_tree(temp_dir)
    print "compress tree(#{temp_dir})\n"
    ls = Dir.glob(File.join(temp_dir, "*")).sort
    ls.each do |path|
      print "compress_tree handling #{path})\n"
      if Dir.exist? path
        print "Found directory #{path}\n"
        compress_tree(path) #recurse
      else
        if File.extname(path).match IMAGE_FILE_EXTENSIONS_PATTERN
          print "Found image #{path}\n"
          destination = ImageHelper.compress_image(path)
        end
      end
    end    
  end
  
  def ingest_tree(document_upload, temp_dir) 
    print "ingest_tree(#{temp_dir})\n"
    # first process all sub-directories
    ls = Dir.glob(File.join(temp_dir, "*")).sort
    ls.each do |path|
      print "ingest_tree considering #{path})\n"
      if Dir.exist? path
        print "Found directory #{path}\n"
        ingest_tree(document_upload, path) #recurse
      end
    end    
    
    # now process this directory if it contains image files
    image_files = Dir.glob(File.join(temp_dir, "*.{"+IMAGE_FILE_EXTENSIONS.join(',')+"}"))
    if image_files.length > 0
      print "Found #{image_files.length} image files in #{temp_dir} -- converting to a work\n"
      convert_to_work(document_upload, temp_dir)
      print "Finished converting files in #{temp_dir} to a work\n"
    end
    print "Finished ingest_tree for #{temp_dir}\n"
    
  end

  WHITELIST =  [
   "title",
   "identifier",
   "description",
   "restrict_scribes",
   "physical_description",
   "document_history",
   "permission_description",
   "location_of_composition",
   "author",
   "transcription_conventions",
   "scribes_can_edit_titles",
   "supports_translation",
   "translation_instructions",
   "pages_are_meaningful",
   "document_set",
   "slug"
  ]

  
  def convert_to_work(document_upload, path)
    print "convert_to_work creating database record for #{path}\n"
    print "\tconvert_to_work owner = #{document_upload.user.login}\n"
    print "\tconvert_to_work collection = #{document_upload.collection.title}\n"
    print "\tconvert_to_work default title = #{File.basename(path).ljust(3,'.')}\n"
    print "\tconvert_to_work looking for metadata.yml in #{File.join(File.dirname(path), 'metadata.yml')}\n"
    
    
    if File.exist? File.join(path, 'metadata.yml')
      yaml = YAML.load_file(File.join(path, 'metadata.yml'))
    elsif File.exist? File.join(path, 'metadata.yaml')
      yaml = YAML.load_file(File.join(path, 'metadata.yaml'))
    else
      print "\tconvert_to_work no metadata.yml file; using default settings\n"
      yaml = nil
    end
          
    print "\tconvert_to_work loaded metadata.yml values \n#{yaml.to_s}\n"
    
#    binding.pry if path == "/tmp/fromthepage_uploads/16/terrell-papers-jpg"
    User.current_user=document_upload.user
    document_sets = []
    if yaml
      yaml.keep_if { |e| WHITELIST.include? e }
      print "\tconvert_to_work whitelisted metadata.yml values \n#{yaml.to_s}\n"
      document_sets = document_sets_from_yaml(yaml, document_upload.collection)
    end
    work = Work.new(yaml)
    work.owner = document_upload.user
    work.collection = document_upload.collection

    work.title = File.basename(path).ljust(3,'.') unless work.title
    if document_upload.ocr
      if Dir.glob(File.join(path, "page*.txt")).count > 0
        work.ocr_correction = true
      else
        print "\tOCR correction specifiied but no files found in #{File.join(path, "page*.txt")}\n"
      end
    end

    work.save!
    
    new_dir_name = File.join(Rails.root,
                             "public",
                             "images",
                             "uploaded",
                             work.id.to_s)
    print "\tconvert_to_work creating #{new_dir_name}\n"
                             <commit_msg>Fixed #1844 by removing outdated assumption<commit_after>require 'image_helper'
require 'open-uri' # TODO: Move elsewhere
namespace :fromthepage do

  desc "Resize image file or directories of image files"
  task :compress_images, [:pathname] => :environment  do  |t,args|
    pathname = args.pathname
    p "compressing #{pathname}"
    
    if Dir.exist? pathname
      ImageHelper.compress_files_in_dir(pathname)
    else
      # this is a single file
      ImageHelper.compress_file(pathname)
    end
  end
  IMAGE_FILE_EXTENSIONS = ['jpg', 'JPG', 'jpeg', 'JPEG', 'png', 'PNG']
  IMAGE_FILE_EXTENSIONS_PATTERN = /jpg|JPG|jpeg|JPEG|png|PNG/
  TIFF_FILE_EXTENSIONS_PATTERN = /tif|TIF|tiff|TIFF/

  desc "Process a document upload"
  task :process_document_upload, [:document_upload_id] => :environment do |t,args|
    require "#{Rails.root}/app/helpers/error_helper"
    include ErrorHelper

    document_upload_id = args.document_upload_id
    print "fetching upload with ID=#{document_upload_id}\n"
    document_upload = DocumentUpload.find document_upload_id
    
    print "found document_upload for \n\tuser=#{document_upload.user.login}, \n\ttarget collection=#{document_upload.collection.title}, \n\tfile=#{document_upload.file}\n"
    
    document_upload.status = DocumentUpload::Status::PROCESSING
    document_upload.save
    
    process_batch(document_upload, File.dirname(document_upload.file.path), document_upload.id.to_s)

    document_upload.status = DocumentUpload::Status::FINISHED
    document_upload.save

    #if the upload processes correctly,
    #remove the uploaded file to prevent filling up the disk
    if document_upload.status = DocumentUpload::Status::FINISHED
      document_upload.remove_file!
      document_upload.save
    end

    if SMTP_ENABLED
      begin
        SystemMailer.upload_succeeded(document_upload).deliver!
        UserMailer.upload_finished(document_upload).deliver!
      rescue StandardError => e
        print "SMTP Failed: Exception: #{e.message}"
      end
    end

  end


  def process_batch(document_upload, path, temp_dir_seed)
    # copy to temp dir
    temp_dir = temp_dir_path(temp_dir_seed)
    copy_to_temp_dir(path, temp_dir)

    # unzip everything
    unzip_tree(temp_dir)
    # extract any pdfs
    unpdf_tree(temp_dir, document_upload.ocr)
    #convert tiffs to jpgs
    untiff_tree(temp_dir)
    # resize files
    compress_tree(temp_dir)
    # ingest
    ingest_tree(document_upload, temp_dir)
    # clean
    clean_tmp_dir(temp_dir)
  end
  
  def clean_tmp_dir(temp_dir)
    print "Removing #{temp_dir}\n"
    FileUtils::rm_r(temp_dir)
  end
  
  def unzip_tree(temp_dir)
    print "unzip_tree(#{temp_dir})\n"
    ls = Dir.glob(File.join(temp_dir, "*"))
    ls.each do |path|
      print "\tunzip_tree considering #{path}\n"
      if Dir.exist? path
        print "Found directory #{path}\n"
        unzip_tree(path) #recurse
      else
        if File.extname(path) == '.ZIP' || File.extname(path) == '.zip'
          print "Found zipfile #{path}\n"
          #unzip and recur
          destination = File.join(File.dirname(path), File.basename(path).sub(File.extname(path),''))
          print "Calling unzip_file(#{path}, #{destination})\n"
          ImageHelper.unzip_file(path, destination)
          unzip_tree(destination)  # recurse
        end
      end
    end
    FileUtils.chmod_R "u=rwx,go=r", temp_dir
  end
  
  def unpdf_tree(temp_dir, ocr)
    print "unpdf_tree(#{temp_dir})\n"
    ls = Dir.glob(File.join(temp_dir, "*"))
    ls.each do |path|
      print "\tunpdf_tree considering #{path})\n"
      if Dir.exist? path
        print "\tunpdf_tree Found directory #{path}\n"
        unpdf_tree(path, ocr) #recurse
      else
        if File.extname(path) == '.PDF' || File.extname(path) == '.pdf'
          print "\t\tunpdf_tree Found pdf #{path}\n"
          #extract 
          destination = ImageHelper.extract_pdf(path, ocr)
          print "\t\tunpdf_tree Extracted to #{destination}\n"
          # copy any metadata.yml to the destination
          metadata_fn = File.join(File.dirname(path), 'metadata.yml')
          if File.exist? metadata_fn
            print "\t\tunpdf_tree Copy #{metadata_fn} to #{destination}\n"
            FileUtils.cp(metadata_fn, destination)
          else
            print "\t\tunpdf_tree No metadata file exists at #{metadata_fn}\n"
          end
        end
      end
    end
  end

  def untiff_tree(temp_dir)
    print "convert tiffs from tree(#{temp_dir})\n"
    ls = Dir.glob(File.join(temp_dir, "*"))
    ls.each do |path|
      print "\tuntiff_tree considering #{path})\n"
      if Dir.exist? path
        print "Found directory #{path}\n"
        untiff_tree(path) #recurse
      else
        if File.extname(path).match TIFF_FILE_EXTENSIONS_PATTERN
          print "Found tiff #{path}\n"
          #convert tiff to jpg
          destination = ImageHelper.convert_tiff(path)
          GC.start
        end
      end
    end

  end

  def compress_tree(temp_dir)
    print "compress tree(#{temp_dir})\n"
    ls = Dir.glob(File.join(temp_dir, "*")).sort
    ls.each do |path|
      print "compress_tree handling #{path})\n"
      if Dir.exist? path
        print "Found directory #{path}\n"
        compress_tree(path) #recurse
      else
        if File.extname(path).match IMAGE_FILE_EXTENSIONS_PATTERN
          print "Found image #{path}\n"
          destination = ImageHelper.compress_image(path)
        end
      end
    end    
  end
  
  def ingest_tree(document_upload, temp_dir) 
    print "ingest_tree(#{temp_dir})\n"
    # first process all sub-directories
    ls = Dir.glob(File.join(temp_dir, "*")).sort
    ls.each do |path|
      print "ingest_tree considering #{path})\n"
      if Dir.exist? path
        print "Found directory #{path}\n"
        ingest_tree(document_upload, path) #recurse
      end
    end    
    
    # now process this directory if it contains image files
    image_files = Dir.glob(File.join(temp_dir, "*.{"+IMAGE_FILE_EXTENSIONS.join(',')+"}"))
    if image_files.length > 0
      print "Found #{image_files.length} image files in #{temp_dir} -- converting to a work\n"
      convert_to_work(document_upload, temp_dir)
      print "Finished converting files in #{temp_dir} to a work\n"
    end
    print "Finished ingest_tree for #{temp_dir}\n"
    
  end

  WHITELIST =  [
   "title",
   "identifier",
   "description",
   "restrict_scribes",
   "physical_description",
   "document_history",
   "permission_description",
   "location_of_composition",
   "author",
   "transcription_conventions",
   "scribes_can_edit_titles",
   "supports_translation",
   "translation_instructions",
   "pages_are_meaningful",
   "document_set",
   "slug"
  ]

  
  def convert_to_work(document_upload, path)
    print "convert_to_work creating database record for #{path}\n"
    print "\tconvert_to_work owner = #{document_upload.user.login}\n"
    print "\tconvert_to_work collection = #{document_upload.collection.title}\n"
    print "\tconvert_to_work default title = #{File.basename(path).ljust(3,'.')}\n"
    print "\tconvert_to_work looking for metadata.yml in #{File.join(File.dirname(path), 'metadata.yml')}\n"
    
    
    if File.exist? File.join(path, 'metadata.yml')
      yaml = YAML.load_file(File.join(path, 'metadata.yml'))
    elsif File.exist? File.join(path, 'metadata.yaml')
      yaml = YAML.load_file(File.join(path, 'metadata.yaml'))
    else
      print "\tconvert_to_work no metadata.yml file; using default settings\n"
      yaml = nil
    end
          
    print "\tconvert_to_work loaded metadata.yml values \n#{yaml.to_s}\n"
    
#    binding.pry if path == "/tmp/fromthepage_uploads/16/terrell-papers-jpg"
    User.current_user=document_upload.user
    document_sets = []
    if yaml
      yaml.keep_if { |e| WHITELIST.include? e }
      print "\tconvert_to_work whitelisted metadata.yml values \n#{yaml.to_s}\n"
      document_sets = document_sets_from_yaml(yaml, document_upload.collection)
    end
    work = Work.new(yaml)
    work.owner = document_upload.user
    work.collection = document_upload.collection

    work.title = File.basename(path).ljust(3,'.') unless work.title
    if document_upload.ocr
      if Dir.glob(File.join(path, "*.txt")).count > 0
        work.ocr_correction = true
      else
        print "\tOCR correction specifiied but no files found in #{File.join(path, "page*.txt")}\n"
      end
    end

    work.save!
    
    new_dir_name = File.join(Rails.root,
                             "public",
                             "images",
                             "uploaded",
                             work.id.to_s)
    print "\tconvert_to_work creating #{new_dir_name}\n"
                             
{'fraction_non_alphanumeric': 0.11258164317634926, 'fraction_numerical': 0.0009166953133952103, 'mean_word_length': 2.8094718463553034, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3479318', 'n_tokens_mistral': 5776, 'n_tokens_neox': 5622, 'n_words': 1221}