<commit_before>require 'aws/s3'
require 'net/sftp'

module DocCollections
  class Upload < Services::Base
    SyncError = Class.new(Error)

    def call(id_or_object)
      doc_collection = find_object(id_or_object)
      check_uniqueness doc_collection.id
      raise Error, "Doc collection #{doc_collection.name} is not generated yet." if doc_collection.generating?
      raise Error, "Doc collection #{doc_collection.name} local path doesn't exist or is empty." unless File.exist?(doc_collection.local_path) && Dir[File.join(doc_collection.local_path, '*')].present?

      # Sync docs
      local_path = doc_collection.local_path.to_s
      local_path << '/' unless local_path.end_with?('/')
      remote_path = local_path.split('/').last << '/'
      s3cmd = Rails.root.join('lib', 's3cmd', 's3cmd')
      s3cmd_options = {
        access_key:              Settings.aws.key,
        secret_key:              Settings.aws.secret,
        quiet:                   true,
        :'delete-removed'     => true,
        :'no-preserve'        => true,
        :'reduced-redundancy' => true
      }
      s3cmd_args = s3cmd_options.map do |k, v|
        "--#{k}".tap do |option|
          option << "=#{v}" unless v == true
        end
      end
      10.tries on: SyncError do
        command = "#{s3cmd} sync #{s3cmd_args.join(' ')} #{local_path} s3://#{Settings.aws.bucket}/#{remote_path}"
        result = system(command)
        raise SyncError, "Sync ended with non-success error status: #{$?}, command: #{command}" unless result
      end

      # Upload zip
      raise Error, "Doc collection #{doc_collection.name} zipfile doesn't exist." unless File.exist?(doc_collection.zipfile)
      # Follow symlinks
      local_file = Pathname.new(doc_collection.zipfile).realpath.to_s
      remote_file = File.join('public_html', File.basename(doc_collection.zipfile))
      Net::SFTP.start(Settings.zip_ftp.host, Settings.zip_ftp.username, password: Settings.zip_ftp.password) do |sftp|
        sftp.upload! local_file, remote_file
      end

      # Set uploaded_at timestamp
      doc_collection.uploaded_at = Time.now
      doc_collection.save!

      # Delete doc collection files
      DocCollections::DeleteFiles.call doc_collection

      # Send notifications
      unless Rails.env.development?
        emails = EmailNotification.by_doc_collection(doc_collection).map(&:email)
        if emails.present?
          Mailer.doc_collection_generated(doc_collection, emails).deliver!
          EmailNotification.delete(doc_collection)
        end
      end

      doc_collection
    end
  end
end<commit_msg>retry upling zip file 3 times<commit_after>require 'aws/s3'
require 'net/sftp'

module DocCollections
  class Upload < Services::Base
    SyncError = Class.new(Error)

    def call(id_or_object)
      doc_collection = find_object(id_or_object)
      check_uniqueness doc_collection.id
      raise Error, "Doc collection #{doc_collection.name} is not generated yet." if doc_collection.generating?
      raise Error, "Doc collection #{doc_collection.name} local path doesn't exist or is empty." unless File.exist?(doc_collection.local_path) && Dir[File.join(doc_collection.local_path, '*')].present?

      # Sync docs
      local_path = doc_collection.local_path.to_s
      local_path << '/' unless local_path.end_with?('/')
      remote_path = local_path.split('/').last << '/'
      s3cmd = Rails.root.join('lib', 's3cmd', 's3cmd')
      s3cmd_options = {
        access_key:              Settings.aws.key,
        secret_key:              Settings.aws.secret,
        quiet:                   true,
        :'delete-removed'     => true,
        :'no-preserve'        => true,
        :'reduced-redundancy' => true
      }
      s3cmd_args = s3cmd_options.map do |k, v|
        "--#{k}".tap do |option|
          option << "=#{v}" unless v == true
        end
      end
      10.tries on: SyncError do
        command = "#{s3cmd} sync #{s3cmd_args.join(' ')} #{local_path} s3://#{Settings.aws.bucket}/#{remote_path}"
        result = system(command)
        raise SyncError, "Sync ended with non-success error status: #{$?}, command: #{command}" unless result
      end

      # Upload zip
      raise Error, "Doc collection #{doc_collection.name} zipfile doesn't exist." unless File.exist?(doc_collection.zipfile)
      # Follow symlinks
      local_file = Pathname.new(doc_collection.zipfile).realpath.to_s
      remote_file = File.join('public_html', File.basename(doc_collection.zipfile))
      Net::SFTP.start(Settings.zip_ftp.host, Settings.zip_ftp.username, password: Settings.zip_ftp.password) do |sftp|
        3.tries on: IOError, Net::SSH::Disconnect do
          sftp.upload! local_file, remote_file
        end
      end

      # Set uploaded_at timestamp
      doc_collection.uploaded_at = Time.now
      doc_collection.save!

      # Delete doc collection files
      DocCollections::DeleteFiles.call doc_collection

      # Send notifications
      unless Rails.env.development?
        emails = EmailNotification.by_doc_collection(doc_collection).map(&:email)
        if emails.present?
          Mailer.doc_collection_generated(doc_collection, emails).deliver!
          EmailNotification.delete(doc_collection)
        end
      end

      doc_collection
    end
  end
end
{'fraction_non_alphanumeric': 0.11295493117103526, 'fraction_numerical': 0.004902885159343767, 'mean_word_length': 2.753715498938429, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1673338', 'n_tokens_mistral': 1636, 'n_tokens_neox': 1591, 'n_words': 346}