<commit_before>require 'digest/md5'
require 'openssl'
require 'uri'

module Raca

  # Represents a single cloud files container. Contains methods for uploading,
  # downloading, collecting stats, listing files, etc.
  #
  # You probably don't want to instantiate this directly,
  # see Raca::Account#containers
  #
  class Container
    MAX_ITEMS_PER_LIST = 10_000
    LARGE_FILE_THRESHOLD = 5_368_709_120 # 5 Gb
    LARGE_FILE_SEGMENT_SIZE = 4_295_000_000 # 4 Gb

    attr_reader :container_name

    def initialize(account, region, container_name, opts = {})
      raise ArgumentError, "The container name must not contain '/'." if container_name['/']
      @account, @region, @container_name = account, region, container_name
      @storage_url = @account.public_endpoint("cloudFiles", region)
      @cdn_url     = @account.public_endpoint("cloudFilesCDN", region)
      @logger = opts[:logger]
      @logger ||= Rails.logger if defined?(Rails)
    end

    # Upload data_or_path (which may be a filename or an IO) to the container, as key.
    #
    # If headers are provided they will be added to to upload request. Use this to
    # manually specify content type, content disposition, CORS headers, etc.
    #
    def upload(key, data_or_path, headers = {})
      case data_or_path
      when StringIO, File
        upload_io(key, data_or_path, data_or_path.size, headers)
      when String
        File.open(data_or_path, "rb") do |io|
          upload_io(key, io, io.stat.size, headers)
        end
      else
        raise ArgumentError, "data_or_path must be an IO with data or filename string"
      end
    end

    # Delete +key+ from the container. If the container is on the CDN, the object will
    # still be served from the CDN until the TTL expires.
    #
    def delete(key)
      log "deleting #{key} from #{container_path}"
      object_path = File.join(container_path, Raca::Util.url_encode(key))
      response = storage_client.delete(object_path)
      (200..299).cover?(response.code.to_i)
    end

    # Remove +key+ from the CDN edge nodes on which it is currently cached. The object is
    # not deleted from the container: as the URL is re-requested, the edge cache will be
    # re-filled with the object currently in the container.
    #
    # This shouldn't be used except when it's really required (e.g. when a piece has to be
    # taken down) because it's expensive: it lodges a support ticket at Akamai. (!)
    #
    def purge_from_akamai(key, email_address)
      log "Requesting #{File.join(container_path, key)} to be purged from the CDN"
      response = cdn_client.delete(
        File.join(container_path, Raca::Util.url_encode(key)),
        'X-Purge-Email' => email_address
      )
      (200..299).cover?(response.code.to_i)
    end<commit_msg>allow IO duck-typing for file uploads<commit_after>require 'digest/md5'
require 'openssl'
require 'uri'

module Raca

  # Represents a single cloud files container. Contains methods for uploading,
  # downloading, collecting stats, listing files, etc.
  #
  # You probably don't want to instantiate this directly,
  # see Raca::Account#containers
  #
  class Container
    MAX_ITEMS_PER_LIST = 10_000
    LARGE_FILE_THRESHOLD = 5_368_709_120 # 5 Gb
    LARGE_FILE_SEGMENT_SIZE = 4_295_000_000 # 4 Gb

    attr_reader :container_name

    def initialize(account, region, container_name, opts = {})
      raise ArgumentError, "The container name must not contain '/'." if container_name['/']
      @account, @region, @container_name = account, region, container_name
      @storage_url = @account.public_endpoint("cloudFiles", region)
      @cdn_url     = @account.public_endpoint("cloudFilesCDN", region)
      @logger = opts[:logger]
      @logger ||= Rails.logger if defined?(Rails)
    end

    # Upload data_or_path (which may be a filename or an IO) to the container, as key.
    #
    # If headers are provided they will be added to to upload request. Use this to
    # manually specify content type, content disposition, CORS headers, etc.
    #
    def upload(key, data_or_path, headers = {})
      if data_or_path.respond_to?(:read) && data_or_path.respond_to?(:size)
        upload_io(key, data_or_path, data_or_path.size, headers)
      elsif !File.file?(data_or_path.to_s)
        raise ArgumentError, "data_or_path must be an IO with data or filename string"
      else
        File.open(data_or_path.to_s, "rb") do |io|
          upload_io(key, io, io.stat.size, headers)
        end
      end
    end

    # Delete +key+ from the container. If the container is on the CDN, the object will
    # still be served from the CDN until the TTL expires.
    #
    def delete(key)
      log "deleting #{key} from #{container_path}"
      object_path = File.join(container_path, Raca::Util.url_encode(key))
      response = storage_client.delete(object_path)
      (200..299).cover?(response.code.to_i)
    end

    # Remove +key+ from the CDN edge nodes on which it is currently cached. The object is
    # not deleted from the container: as the URL is re-requested, the edge cache will be
    # re-filled with the object currently in the container.
    #
    # This shouldn't be used except when it's really required (e.g. when a piece has to be
    # taken down) because it's expensive: it lodges a support ticket at Akamai. (!)
    #
    def purge_from_akamai(key, email_address)
      log "Requesting #{File.join(container_path, key)} to be purged from the CDN"
      response = cdn_client.delete(
        File.join(container_path, Raca::Util.url_encode(key)),
        'X-Purge-Email' => email_address
      )
      (200..299).cover?(response.code.to_i)
    end
{'fraction_non_alphanumeric': 0.09131049099258212, 'fraction_numerical': 0.01412928293889085, 'mean_word_length': 3.1670345842531273, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5682173', 'n_tokens_mistral': 1830, 'n_tokens_neox': 1740, 'n_words': 576}