<commit_before>module Astrails
  module Safe
    class S3 < Sink
      MAX_S3_FILE_SIZE = 5368709120

      protected

      def active?
        bucket && key && secret
      end

      def path
        @path ||= expand(config[:s3, :path] || config[:local, :path] || ":kind/:id")
      end

      def save
        # FIXME: user friendly error here :)
        raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path

        puts "Uploading #{bucket}:#{full_path}" if $_VERBOSE || $DRY_RUN
        unless $DRY_RUN || $LOCAL
          if File.stat(@backup.path).size > MAX_S3_FILE_SIZE
            STDERR.puts "ERROR: File size exceeds maximum allowed for upload to S3 (#{MAX_S3_FILE_SIZE}): #{@backup.path}"
            return
          end<commit_msg>fixes s3 file size limit is now 5tb with multi file upload. aws-sdk automatically uses multi upload for files > 5gb.<commit_after>module Astrails
  module Safe
    class S3 < Sink
      MAX_S3_FILE_SIZE = 5_368_709_120 * 1024 # 5TB

      protected

      def active?
        bucket && key && secret
      end

      def path
        @path ||= expand(config[:s3, :path] || config[:local, :path] || ":kind/:id")
      end

      def save
        # FIXME: user friendly error here :)
        raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path

        puts "Uploading #{bucket}:#{full_path}" if $_VERBOSE || $DRY_RUN
        unless $DRY_RUN || $LOCAL
          if File.stat(@backup.path).size > MAX_S3_FILE_SIZE
            STDERR.puts "ERROR: File size exceeds maximum allowed for upload to S3 (#{MAX_S3_FILE_SIZE}): #{@backup.path}"
            return
          end
{'fraction_non_alphanumeric': 0.10557888422315537, 'fraction_numerical': 0.02519496100779844, 'mean_word_length': 2.376518218623482, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '625833', 'n_tokens_mistral': 534, 'n_tokens_neox': 506, 'n_words': 156}