<commit_before>require 'aws/s3'
require 'net/sftp'

module Services
  module DocCollections
    class UploadFiles < Services::Base
      SyncError = Class.new(Error)

      check_uniqueness!

      def call(id_or_object)
        doc_collection = find_object(id_or_object)
        raise Error, "Doc collection #{doc_collection.name} is not generated yet." if doc_collection.generating?
        raise Error, "Doc collection #{doc_collection.name} local path doesn't exist or is empty." unless File.exist?(doc_collection.local_path) && Dir[File.join(doc_collection.local_path, '*')].present?

        # Sync docs
        local_path = doc_collection.local_path.to_s
        local_path << '/' unless local_path.end_with?('/')
        remote_path = local_path.split('/').last << '/'
        s3cmd = Rails.root.join('lib', 's3cmd', 's3cmd')
        s3cmd_options = {
          access_key:              Settings.aws.key,
          secret_key:              Settings.aws.secret,
          verbose:                 true,
          :'delete-removed'     => true,
          :'no-preserve'        => true,
          :'reduced-redundancy' => true
        }
        s3cmd_args = s3cmd_options.map do |k, v|
          "--#{k}".tap do |option|
            option << "=#{v}" unless v == true
          end
        end
        3.tries on: SyncError do
          result = system("#{s3cmd} sync #{s3cmd_args.join(' ')} #{local_path} s3://#{Settings.aws.bucket}/#{remote_path}")
          raise SyncError, "Sync ended with non-success error status: #{$?}" unless result
        end

        # Upload zip
        raise Error, "Doc collection #{doc_collection.name} zipfile doesn't exist." unless File.exist?(doc_collection.zipfile)
        # Follow symlinks
        local_file = Pathname.new(doc_collection.zipfile).realpath.to_s
        remote_file = File.join('public_html', File.basename(doc_collection.zipfile))
        Net::SFTP.start(Settings.zip_ftp.host, Settings.zip_ftp.username, password: Settings.zip_ftp.password) do |sftp|
          sftp.upload! local_file, remote_file
        end

        # Set uploaded_at timestamp
        doc_collection.uploaded_at = Time.now
        doc_collection.save!

        # Delete doc collection files
        Services::DocCollections::DeleteFiles.call doc_collection
<commit_msg>Output complete s3cmd command when it fails<commit_after>require 'aws/s3'
require 'net/sftp'

module Services
  module DocCollections
    class UploadFiles < Services::Base
      SyncError = Class.new(Error)

      check_uniqueness!

      def call(id_or_object)
        doc_collection = find_object(id_or_object)
        raise Error, "Doc collection #{doc_collection.name} is not generated yet." if doc_collection.generating?
        raise Error, "Doc collection #{doc_collection.name} local path doesn't exist or is empty." unless File.exist?(doc_collection.local_path) && Dir[File.join(doc_collection.local_path, '*')].present?

        # Sync docs
        local_path = doc_collection.local_path.to_s
        local_path << '/' unless local_path.end_with?('/')
        remote_path = local_path.split('/').last << '/'
        s3cmd = Rails.root.join('lib', 's3cmd', 's3cmd')
        s3cmd_options = {
          access_key:              Settings.aws.key,
          secret_key:              Settings.aws.secret,
          verbose:                 true,
          :'delete-removed'     => true,
          :'no-preserve'        => true,
          :'reduced-redundancy' => true
        }
        s3cmd_args = s3cmd_options.map do |k, v|
          "--#{k}".tap do |option|
            option << "=#{v}" unless v == true
          end
        end
        3.tries on: SyncError do
          command = "#{s3cmd} sync #{s3cmd_args.join(' ')} #{local_path} s3://#{Settings.aws.bucket}/#{remote_path}"
          result = system(command)
          raise SyncError, "Sync ended with non-success error status: #{$?}, command: #{command}" unless result
        end

        # Upload zip
        raise Error, "Doc collection #{doc_collection.name} zipfile doesn't exist." unless File.exist?(doc_collection.zipfile)
        # Follow symlinks
        local_file = Pathname.new(doc_collection.zipfile).realpath.to_s
        remote_file = File.join('public_html', File.basename(doc_collection.zipfile))
        Net::SFTP.start(Settings.zip_ftp.host, Settings.zip_ftp.username, password: Settings.zip_ftp.password) do |sftp|
          sftp.upload! local_file, remote_file
        end

        # Set uploaded_at timestamp
        doc_collection.uploaded_at = Time.now
        doc_collection.save!

        # Delete doc collection files
        Services::DocCollections::DeleteFiles.call doc_collection

{'fraction_non_alphanumeric': 0.11578722495193335, 'fraction_numerical': 0.004913480025635548, 'mean_word_length': 2.494029850746269, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3390440', 'n_tokens_mistral': 1415, 'n_tokens_neox': 1379, 'n_words': 306}