<commit_before>module Jekyll
  module S3
    class Uploader
      def self.run(site_dir, config, in_headless_mode = false)
        puts "Deploying _site/* to #{config['s3_bucket']}"

        s3 = AWS::S3.new(:access_key_id => config['s3_id'],
                         :secret_access_key => config['s3_secret'],
                         :s3_endpoint => Endpoint.new(config['s3_endpoint']).hostname )

        new_files_count, changed_files_count, changed_files = upload_files(
          s3, config, site_dir
        )

        deleted_files_count = remove_superfluous_files(
          s3, config['s3_bucket'], site_dir, in_headless_mode)

        print_done_report config

        [new_files_count, changed_files_count, deleted_files_count, changed_files]
      end

      private

      def self.print_done_report(config)
        bucket_name = config['s3_bucket']
        website_hostname_suffix = Endpoint.new(config['s3_endpoint']).website_hostname
        website_hostname_with_bucket =
          "%s.%s" % [bucket_name, website_hostname_suffix]
        puts "Done! Go visit: http://#{website_hostname_with_bucket}/index.html"
      end

      def self.upload_files(s3, config, site_dir)
        changed_files, new_files = DiffHelper.resolve_files_to_upload(
          s3.buckets[config['s3_bucket']], site_dir)
        to_upload = changed_files + new_files
        if to_upload.empty?
          puts "No new or changed files to upload"
        else
          pre_upload_report = []
          pre_upload_report << "Uploading"
          pre_upload_report << "#{new_files.length} new" if new_files.length > 0
          pre_upload_report << "and" if changed_files.length > 0 and new_files.length > 0
          pre_upload_report << "#{changed_files.length} changed" if changed_files.length > 0
          pre_upload_report << "file(s)"
          puts pre_upload_report.join(' ')
          to_upload.each do |f|
            upload_file(f, s3, config, site_dir)
          end
        end
        [new_files.length, changed_files.length, changed_files]
      end

      def self.upload_file(file, s3, config, site_dir)
        Retry.run_with_retry do
          upload = Upload.new(file, s3, config, site_dir)

          if upload.perform!
            puts "Upload #{upload.details}: Success!"
          else
            puts "Upload #{upload.details}: FAILURE!"
          end
        end
      end

      def self.remove_superfluous_files(s3, s3_bucket_name, site_dir, in_headless_mode)
        remote_files = s3.buckets[s3_bucket_name].objects.map { |f| f.key }
        local_files = load_all_local_files(site_dir)
        files_to_delete = remote_files - local_files
        deleted_files_count = 0
        if in_headless_mode
          files_to_delete.each { |s3_object_key|
            delete_s3_object s3, s3_bucket_name, s3_object_key
            deleted_files_count += 1
          }
        else
          Keyboard.if_user_confirms_delete(files_to_delete) { |s3_object_key|
            delete_s3_object s3, s3_bucket_name, s3_object_key
            deleted_files_count += 1
          }
        end
        deleted_files_count
      end

      def self.delete_s3_object(s3, s3_bucket_name, s3_object_key)
        Retry.run_with_retry do
          s3.buckets[s3_bucket_name].objects[s3_object_key].delete
          puts("Delete #{s3_object_key}: Success!")
        end
      end

      def self.load_all_local_files(site_dir)
        Dir[site_dir + '/**/{*,.*}'].
          delete_if { |f| File.directory?(f) }.
          map { |f| f.gsub(site_dir + '/', '') }
      end
    end
  end
end<commit_msg>Load dotfiles also with Ruby 2.0.0<commit_after>module Jekyll
  module S3
    class Uploader
      def self.run(site_dir, config, in_headless_mode = false)
        puts "Deploying _site/* to #{config['s3_bucket']}"

        s3 = AWS::S3.new(:access_key_id => config['s3_id'],
                         :secret_access_key => config['s3_secret'],
                         :s3_endpoint => Endpoint.new(config['s3_endpoint']).hostname )

        new_files_count, changed_files_count, changed_files = upload_files(
          s3, config, site_dir
        )

        deleted_files_count = remove_superfluous_files(
          s3, config['s3_bucket'], site_dir, in_headless_mode)

        print_done_report config

        [new_files_count, changed_files_count, deleted_files_count, changed_files]
      end

      private

      def self.print_done_report(config)
        bucket_name = config['s3_bucket']
        website_hostname_suffix = Endpoint.new(config['s3_endpoint']).website_hostname
        website_hostname_with_bucket =
          "%s.%s" % [bucket_name, website_hostname_suffix]
        puts "Done! Go visit: http://#{website_hostname_with_bucket}/index.html"
      end

      def self.upload_files(s3, config, site_dir)
        changed_files, new_files = DiffHelper.resolve_files_to_upload(
          s3.buckets[config['s3_bucket']], site_dir)
        to_upload = changed_files + new_files
        if to_upload.empty?
          puts "No new or changed files to upload"
        else
          pre_upload_report = []
          pre_upload_report << "Uploading"
          pre_upload_report << "#{new_files.length} new" if new_files.length > 0
          pre_upload_report << "and" if changed_files.length > 0 and new_files.length > 0
          pre_upload_report << "#{changed_files.length} changed" if changed_files.length > 0
          pre_upload_report << "file(s)"
          puts pre_upload_report.join(' ')
          to_upload.each do |f|
            upload_file(f, s3, config, site_dir)
          end
        end
        [new_files.length, changed_files.length, changed_files]
      end

      def self.upload_file(file, s3, config, site_dir)
        Retry.run_with_retry do
          upload = Upload.new(file, s3, config, site_dir)

          if upload.perform!
            puts "Upload #{upload.details}: Success!"
          else
            puts "Upload #{upload.details}: FAILURE!"
          end
        end
      end

      def self.remove_superfluous_files(s3, s3_bucket_name, site_dir, in_headless_mode)
        remote_files = s3.buckets[s3_bucket_name].objects.map { |f| f.key }
        local_files = load_all_local_files(site_dir)
        files_to_delete = remote_files - local_files
        deleted_files_count = 0
        if in_headless_mode
          files_to_delete.each { |s3_object_key|
            delete_s3_object s3, s3_bucket_name, s3_object_key
            deleted_files_count += 1
          }
        else
          Keyboard.if_user_confirms_delete(files_to_delete) { |s3_object_key|
            delete_s3_object s3, s3_bucket_name, s3_object_key
            deleted_files_count += 1
          }
        end
        deleted_files_count
      end

      def self.delete_s3_object(s3, s3_bucket_name, s3_object_key)
        Retry.run_with_retry do
          s3.buckets[s3_bucket_name].objects[s3_object_key].delete
          puts("Delete #{s3_object_key}: Success!")
        end
      end

      def self.load_all_local_files(site_dir)
        Dir.glob(site_dir + '/**/*', File::FNM_DOTMATCH).
          delete_if { |f| File.directory?(f) }.
          map { |f| f.gsub(site_dir + '/', '') }
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.09321916862311835, 'fraction_numerical': 0.013672144731390692, 'mean_word_length': 2.3825315273236805, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6855101', 'n_tokens_mistral': 2358, 'n_tokens_neox': 2338, 'n_words': 429}