<commit_before>require "open-uri"
require "yaml"
require "digest/md5"
require "fileutils"
require "tmpdir"
require "openssl"

require "aws-sdk-core"
require "s3_meta_sync/zip"

module S3MetaSync
  class Syncer
    DEFAULT_REGION = 'us-east-1'

    def initialize(config)
      @config = config
    end

    def sync(source, destination)
      raise ArgumentError if source.end_with?("/") or destination.end_with?("/")

      if destination.include?(":")
        @bucket, destination = destination.split(":")
        upload(source, destination)
      else
        if url?(source)
          @bucket = nil
          source = source
        else
          @bucket, source = source.split(":")
        end
        download(source, destination)
      end
    end

    private

    def upload(source, destination)
      corrupted = consume_corrupted_files(source)
      remote_meta = begin
        download_meta(destination)
      rescue RemoteWithoutMeta
        log "Remote has no #{META_FILE}, uploading everything", true
        {files: {}}
      end
      local_files = generate_meta(source)[:files]
      remote_files = remote_meta[:files]
      upload = if @config[:zip] == remote_meta[:zip]
        local_files.select { |path, md5| remote_files[path] != md5 || corrupted.include?(path) }
      else
        local_files
      end.map(&:first)
      delete = remote_files.keys - local_files.keys
      log "Uploading: #{upload.size} Deleting: #{delete.size}", true

      upload_files(source, destination, upload)
      delete_remote_files(destination, delete)
      upload_file(source, META_FILE, destination)
    end

    def download(source, destination)
      remote_meta = download_meta(source)
      local_files = ((@config[:no_local_changes] && read_meta(destination)) || meta_data(destination))[:files]

      download = remote_meta[:files].select { |path, md5| local_files[path] != md5 }.map(&:first)
      delete = local_files.keys - remote_meta[:files].keys

      log "Downloading: #{download.size} Deleting: #{delete.size}", true

      if download.any? || delete.any?
        mktmpdir do |staging_area|
          FileUtils.mkdir_p(destination)
          copy_content(destination, staging_area)
          download_files(source, staging_area, download, remote_meta[:zip])
          delete_local_files(staging_area, delete)
          delete_empty_folders(staging_area)
          store_meta(staging_area, remote_meta)

          verify_integrity!(staging_area, destination, download, remote_meta[:files])
          self.class.swap_in_directory(destination, staging_area)
          FileUtils.mkdir(staging_area) # mktmpdir tries to remove this directory
        end
      end
    end

    def mktmpdir
      dir = Dir.mktmpdir("s3ms_")
      log "Tmp folder: #{dir}"
      yield dir
    ensure
      if dir
        log "FileUtils.remove_entry #{dir}: #{FileUtils.remove_entry(dir, true)}"
        # Sometimes remove_entry fails, trying really hard:
        log "rm -rf #{dir}: #{`rm -rf #{dir}`.inspect}"
      end<commit_msg>Rename method<commit_after>require "open-uri"
require "yaml"
require "digest/md5"
require "fileutils"
require "tmpdir"
require "openssl"

require "aws-sdk-core"
require "s3_meta_sync/zip"

module S3MetaSync
  class Syncer
    DEFAULT_REGION = 'us-east-1'

    def initialize(config)
      @config = config
    end

    def sync(source, destination)
      raise ArgumentError if source.end_with?("/") or destination.end_with?("/")

      if destination.include?(":")
        @bucket, destination = destination.split(":")
        upload(source, destination)
      else
        if url?(source)
          @bucket = nil
          source = source
        else
          @bucket, source = source.split(":")
        end
        download(source, destination)
      end
    end

    private

    def upload(source, destination)
      corrupted = consume_corrupted_files(source)
      remote_meta = begin
        download_meta(destination)
      rescue RemoteWithoutMeta
        log "Remote has no #{META_FILE}, uploading everything", true
        {files: {}}
      end
      local_files = generate_meta(source)[:files]
      remote_files = remote_meta[:files]
      upload = if @config[:zip] == remote_meta[:zip]
        local_files.select { |path, md5| remote_files[path] != md5 || corrupted.include?(path) }
      else
        local_files
      end.map(&:first)
      delete = remote_files.keys - local_files.keys
      log "Uploading: #{upload.size} Deleting: #{delete.size}", true

      upload_files(source, destination, upload)
      delete_remote_files(destination, delete)
      upload_file(source, META_FILE, destination)
    end

    def download(source, destination)
      remote_meta = download_meta(source)
      local_files = ((@config[:no_local_changes] && read_meta(destination)) || meta_data(destination))[:files]

      download = remote_meta[:files].select { |path, md5| local_files[path] != md5 }.map(&:first)
      delete = local_files.keys - remote_meta[:files].keys

      log "Downloading: #{download.size} Deleting: #{delete.size}", true

      if download.any? || delete.any?
        make_temporary_folder do |staging_area|
          FileUtils.mkdir_p(destination)
          copy_content(destination, staging_area)
          download_files(source, staging_area, download, remote_meta[:zip])
          delete_local_files(staging_area, delete)
          delete_empty_folders(staging_area)
          store_meta(staging_area, remote_meta)

          verify_integrity!(staging_area, destination, download, remote_meta[:files])
          self.class.swap_in_directory(destination, staging_area)
          FileUtils.mkdir(staging_area) # mktmpdir tries to remove this directory
        end
      end
    end

    def make_temporary_folder
      dir = Dir.mktmpdir("s3ms_")
      log "Tmp folder: #{dir}"
      yield dir
    ensure
      if dir
        log "FileUtils.remove_entry #{dir}: #{FileUtils.remove_entry(dir, true)}"
        # Sometimes remove_entry fails, trying really hard:
        log "rm -rf #{dir}: #{`rm -rf #{dir}`.inspect}"
      end
{'fraction_non_alphanumeric': 0.1103086014445174, 'fraction_numerical': 0.002954694681549573, 'mean_word_length': 2.9208494208494207, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4595212', 'n_tokens_mistral': 1829, 'n_tokens_neox': 1794, 'n_words': 412}