<commit_before>require 'tempfile'
require 'zlib'

module S3Website
  class Upload
    attr_reader :config, :file, :path, :full_path, :s3
    BLACKLISTED_FILES = ['s3_website.yml']

    def initialize(path, s3, config, site_dir)
      raise "May not upload #{path}, because it's blacklisted" if Upload.is_blacklisted(path, config)
      @path = path
      @full_path = "#{site_dir}/#{path}"
      @file = File.open("#{site_dir}/#{path}")
      @s3 = s3
      @config = config
    end

    def perform!
      success = s3.buckets[config['s3_bucket']].objects[path].write(upload_file, upload_options)
      upload_file.close
      success
    end

    def details
      "#{path}#{" [gzipped]" if gzip?}#{" [max-age=#{max_age}]" if cache_control?}"
    end

    def self.is_blacklisted(path, config)
      [
        config['exclude_from_upload'],
        BLACKLISTED_FILES
      ].flatten.compact.any? do |blacklisted_file|
        Regexp.new(blacklisted_file).match path
      end
    end

    private

    def upload_file
      @upload_file ||= gzip? ? gzipped_file : file
    end

    def gzip?
      return false unless !!config['gzip']

      extensions = config['gzip'].is_a?(Array) ? config['gzip'] : S3Website::DEFAULT_GZIP_EXTENSIONS
      extensions.include?(File.extname(path))
    end

    def gzipped_file
      tempfile = Tempfile.new(File.basename(path))
      tempfile.binmode

      gz = Zlib::GzipWriter.new(tempfile, Zlib::BEST_COMPRESSION, Zlib::DEFAULT_STRATEGY)

      gz.mtime = File.mtime(full_path)
      gz.orig_name = File.basename(path)
      gz.write(file.read)

      gz.flush
      tempfile.flush

      gz.close
      tempfile.open

      tempfile
    end

    def upload_options
      opts = {
        :content_type => mime_type,
        :reduced_redundancy => config['s3_reduced_redundancy']
      }

      opts[:content_type] = "text/html; charset=utf-8" if mime_type == 'text/html'
      opts[:content_encoding] = "gzip" if gzip?
      opts[:cache_control] = cache_control_value if cache_control?

      opts
    end

    def cache_control_value
      if max_age == 0
        "no-cache, max-age=0"
      else<commit_msg>append charset=utf-8 to both text/html and application/json<commit_after>require 'tempfile'
require 'zlib'

module S3Website
  class Upload
    attr_reader :config, :file, :path, :full_path, :s3
    BLACKLISTED_FILES = ['s3_website.yml']

    def initialize(path, s3, config, site_dir)
      raise "May not upload #{path}, because it's blacklisted" if Upload.is_blacklisted(path, config)
      @path = path
      @full_path = "#{site_dir}/#{path}"
      @file = File.open("#{site_dir}/#{path}")
      @s3 = s3
      @config = config
    end

    def perform!
      success = s3.buckets[config['s3_bucket']].objects[path].write(upload_file, upload_options)
      upload_file.close
      success
    end

    def details
      "#{path}#{" [gzipped]" if gzip?}#{" [max-age=#{max_age}]" if cache_control?}"
    end

    def self.is_blacklisted(path, config)
      [
        config['exclude_from_upload'],
        BLACKLISTED_FILES
      ].flatten.compact.any? do |blacklisted_file|
        Regexp.new(blacklisted_file).match path
      end
    end

    private

    def upload_file
      @upload_file ||= gzip? ? gzipped_file : file
    end

    def gzip?
      return false unless !!config['gzip']

      extensions = config['gzip'].is_a?(Array) ? config['gzip'] : S3Website::DEFAULT_GZIP_EXTENSIONS
      extensions.include?(File.extname(path))
    end

    def gzipped_file
      tempfile = Tempfile.new(File.basename(path))
      tempfile.binmode

      gz = Zlib::GzipWriter.new(tempfile, Zlib::BEST_COMPRESSION, Zlib::DEFAULT_STRATEGY)

      gz.mtime = File.mtime(full_path)
      gz.orig_name = File.basename(path)
      gz.write(file.read)

      gz.flush
      tempfile.flush

      gz.close
      tempfile.open

      tempfile
    end

    def upload_options
      opts = {
        :content_type => mime_type,
        :reduced_redundancy => config['s3_reduced_redundancy']
      }

      opts[:content_type] = "#{mime_type}; charset=utf-8" if ['text/html', 'application/json'].include? mime_type
      opts[:content_encoding] = "gzip" if gzip?
      opts[:cache_control] = cache_control_value if cache_control?

      opts
    end

    def cache_control_value
      if max_age == 0
        "no-cache, max-age=0"
      else
{'fraction_non_alphanumeric': 0.12052042912577038, 'fraction_numerical': 0.0061629764893859846, 'mean_word_length': 2.7808455565142363, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6684840', 'n_tokens_mistral': 1552, 'n_tokens_neox': 1477, 'n_words': 294}