<commit_before>require 'middleman-core'
require 'fog'
require 'parallel'
require 'digest/md5'
require 'middleman-s3_sync/version'
require 'middleman-s3_sync/commands'

::Middleman::Extensions.register(:s3_sync, '>= 3.0.0') do
  require 'middleman-s3_sync/extension'
  ::Middleman::S3Sync
end

module Middleman
  module S3Sync
    class << self
      def sync
        puts "Gathering local files."

        local_files = (Dir[options.build_dir + "/**/*"] + Dir[options.build_dir + "/**/.*"])
          .reject { |f| File.directory?(f) }
          .map { |f| f.gsub(/^#{options.build_dir}\//, '') }
        puts "Gathering remote files from #{options.bucket}"
        remote_files = bucket.files.map { |f| f.key }

        if options.force
          files_to_push = local_files
        else
          # First pass on the set of files to work with.
          puts "Determine files to add to #{options.bucket}."
          files_to_push = local_files - remote_files
          if options.delete
            puts "Determine which files to delete from #{options.bucket}"
            files_to_delete = remote_files - local_files
          end
          files_to_evaluate = local_files - files_to_push

          # No need to evaluate the files that are newer on S3 than the local files.
          puts "Determine which local files are newer than their S3 counterparts"
          files_to_reject = []
          Parallel.each(files_to_evaluate, :in_threads => 4) do |f|
            print '.'
            local_mtime = File.mtime("#{options.build_dir}/#{f}")
            remote_mtime = s3_files.get(f).last_modified
            files_to_reject << f if remote_mtime >= local_mtime
          end

          files_to_evaluate = files_to_evaluate - files_to_reject

          # Are the files different? Use MD5 to see
          if (files_to_evaluate.size > 0)
            puts "\n\nDetermine which remaining files are actually different than their S3 counterpart."
            Parallel.each(files_to_evaluate, :in_threads => 4) do |f|
              print '.'
              local_md5 = Digest::MD5.hexdigest(File.read("#{options.build_dir}/#{f}"))
              remote_md5 = s3_files.get(f).etag
              files_to_push << f if local_md5 != remote_md5
            end
          end
        end

        if files_to_push.size > 0
          puts "\n\nReady to apply updates to #{options.bucket}."
          files_to_push.each do |f|
            if remote_files.include?(f)
              puts "Updating #{f}"
              file = s3_files.get(f)
              file.body = File.open("#{options.build_dir}/#{f}")
              file.public = true
              file.content_type = MIME::Types.of(f).first
              if policy = options.caching_policy_for(file.content_type)
                file.cache_control = policy.cache_control if policy.cache_control
                file.expires = policy.expires if policy.expires
              end

              file.save
            else
              puts "Creating #{f}"
              file_hash = {
                :key => f,
                :body => File.open("#{options.build_dir}/#{f}"),
                :public => true,
                :acl => 'public-read',
                :content_type => MIME::Types.of(f).first
              }

              # Add cache-control headers
              if policy = options.caching_policy_for(file_hash[:content_type])
                file_hash[:cache_control] = policy.cache_control if policy.cache_control
                file_hash[:expires] = policy.expires if policy.expires
              end

              file = bucket.files.create(file_hash)
            end
          end
        else
          puts "\n\nNo files to update."
        end

        if options.delete
          files_to_delete.each do |f|
            puts "Deleting #{f}"
            file = s3_files.get(f)
            file.destroy
          end
        end
      end

      protected
      def connection
        @connection ||= Fog::Storage.new({
          :provider => 'AWS',
          :aws_access_key_id => options.aws_access_key_id,
          :aws_secret_access_key => options.aws_secret_access_key,
          :region => options.region
        })
      end

      def bucket
        @bucket ||= connection.directories.get(options.bucket)
      end

      def s3_files
        @s3_files ||= bucket.files
      end
    end
  end
end
<commit_msg>Refactoring how the list of files to process is built and processed.<commit_after>require 'middleman-core'
require 'fog'
require 'parallel'
require 'digest/md5'
require 'middleman-s3_sync/version'
require 'middleman-s3_sync/commands'

::Middleman::Extensions.register(:s3_sync, '>= 3.0.0') do
  require 'middleman-s3_sync/extension'
  ::Middleman::S3Sync
end

module Middleman
  module S3Sync
    class << self
      def sync
        if files_to_create.empty? && files_to_update.empty? && files_to_delete.empty?
          puts "\nAll S3 files are up to date."
          return
        end

        puts "\nReady to apply updates to #{options.bucket}."

        files_to_create.each do |f|
          puts "Creating #{f}"
          file_hash = {
            :key => f,
            :body => File.open(local_path(f)),
            :public => true,
            :acl => 'public-read',
            :content_type => MIME::Types.of(f).first
          }

          # Add cache-control headers
          if policy = options.caching_policy_for(file_hash[:content_type])
            file_hash[:cache_control] = policy.cache_control if policy.cache_control
            file_hash[:expires] = policy.expires if policy.expires
          end

          bucket.files.create(file_hash)
        end

        files_to_update.each do |f|
          puts "Updating #{f}"
          file = s3_files.get(f)
          file.body = File.open(local_path(f))
          file.public = true
          file.content_type = MIME::Types.of(f).first
          if policy = options.caching_policy_for(file.content_type)
            file.cache_control = policy.cache_control if policy.cache_control
            file.expires = policy.expires if policy.expires
          end

          file.save
        end

        files_to_delete.each do |f|
          puts "Deleting #{f}"
          if file = s3_files.get(f)
            file.destroy
          end
        end
      end

      protected
      def connection
        @connection ||= Fog::Storage.new({
          :provider => 'AWS',
          :aws_access_key_id => options.aws_access_key_id,
          :aws_secret_access_key => options.aws_secret_access_key,
          :region => options.region
        })
      end

      def bucket
        @bucket ||= connection.directories.get(options.bucket)
      end

      def s3_files
        @s3_files ||= bucket.files
      end

      def remote_files
        @remote_files ||= begin
                            puts "Gathering remote files from #{options.bucket}"
                            bucket.files.map { |f| f.key }
                          end
      end
      def local_files
        @local_files ||= begin
                           puts "Gathering local files."
                           (Dir[build_dir + "/**/*"] + Dir[build_dir + "/**/.*"])
                             .reject { |f| File.directory?(f) }
                             .map { |f| f.gsub(/^#{build_dir}\//, '') }
                         end
      end

      def files_to_delete
        @files_to_delete ||= begin
                               if options.delete
                                 puts "\nDetermine which files to delete from #{options.bucket}"
                                 remote_files - local_files
                               else
                                 []
                               end
                             end
      end

      def files_to_create
        @files_to_create ||= begin
                               puts "Determine files to add to #{options.bucket}."
                               local_files - remote_files
                             end
      end

      def files_to_evaluate
        @files_to_evaluate ||= begin
                                 local_files - files_to_create
                               end
      end

      def files_to_update
        return files_to_evaluate if options.force

        @files_to_update ||= begin
                               puts "Determine which local files to update their S3 counterparts"
                               files_to_update = []
                               Parallel.each(files_to_evaluate, :in_threads => 4) do |f|
                                 print '.'
                                 remote_file = s3_files.get(f)
                                 local_mtime = File.mtime(local_path(f))
                                 remote_mtime = remote_file.last_modified
                                 if remote_mtime < local_mtime
                                   local_md5 = Digest::MD5.hexdigest(File.read(local_path(f)))
                                   remote_md5 = remote_file.etag
                                   files_to_update << f if local_md5 != remote_md5
                                 end
                               end
                               puts ""
                               files_to_update
                             end
      end

      def local_path(f)
        "#{build_dir}/#{f}"
      end

      def build_dir
        @build_dir ||= options.build_dir
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.09082268454958285, 'fraction_numerical': 0.0054916041820678, 'mean_word_length': 1.4476608942879297, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '481699', 'n_tokens_mistral': 2646, 'n_tokens_neox': 2595, 'n_words': 595}