<commit_before>require 'httpclient'
require 's3'

module Tootsie
  
  class IncompatibleOutputError < Exception; end
  
  class Output
    
    def initialize(url)
      @url = url
      @temp_file = Tempfile.new('tootsie')
      @temp_file.close
      @file_name = @temp_file.path
      @logger = Application.get.logger
    end
    
    # Put data into the output. Options:
    #
    # * +:content_type+ - content type of the stored data.
    #
    def put!(options = {})
      @logger.info("Storing #{@url}")
      case @url
        when /^file:(.*)/
          FileUtils.cp(@temp_file.path, $1)
        when /^s3:.*/
          s3_options = S3Utilities.parse_uri(@url)
          bucket_name, path = s3_options[:bucket], s3_options[:key]
          File.open(@temp_file.path, 'r') do |file|
            s3_service = Tootsie::Application.get.s3_service
            begin
              object = s3_service.buckets.find(bucket_name).objects.build(path)
              object.acl = s3_options[:acl] || :private
              object.content_type = s3_options[:content_type]
              object.content_type ||= @content_type if @content_type
              object.storage_class = s3_options[:storage_class] || :standard
              object.content = file
              object.save
              @result_url = object.url
            rescue ::S3::Error::NoSuchBucket
              raise IncompatibleOutputError, "Bucket #{bucket_name} not found"
            end
          end
        when /^http(s?):\/\//
          File.open(@temp_file.path, 'wb') do |file|
            HTTPClient.new.get_content(@url) do |chunk|
              file << chunk
            end
          end
        else
          raise IncompatibleOutputError, "Don't know to store output URL: #{@url}"
      end
    end
    
    def close
      @temp_file.unlink
    end
    <commit_msg>Fix output to HTTP URLs to POST data correctly.<commit_after>require 'httpclient'
require 's3'

module Tootsie
  
  class IncompatibleOutputError < Exception; end
  
  class Output
    
    def initialize(url)
      @url = url
      @temp_file = Tempfile.new('tootsie')
      @temp_file.close
      @file_name = @temp_file.path
      @logger = Application.get.logger
    end
    
    # Put data into the output. Options:
    #
    # * +:content_type+ - content type of the stored data.
    #
    def put!(options = {})
      @logger.info("Storing #{@url}")
      case @url
        when /^file:(.*)/
          FileUtils.cp(@temp_file.path, $1)
        when /^s3:.*/
          s3_options = S3Utilities.parse_uri(@url)
          bucket_name, path = s3_options[:bucket], s3_options[:key]
          File.open(@temp_file.path, 'r') do |file|
            s3_service = Tootsie::Application.get.s3_service
            begin
              object = s3_service.buckets.find(bucket_name).objects.build(path)
              object.acl = s3_options[:acl] || :private
              object.content_type = s3_options[:content_type]
              object.content_type ||= @content_type if @content_type
              object.storage_class = s3_options[:storage_class] || :standard
              object.content = file
              object.save
              @result_url = object.url
            rescue ::S3::Error::NoSuchBucket
              raise IncompatibleOutputError, "Bucket #{bucket_name} not found"
            end
          end
        when /^http(s?):\/\//
          File.open(@temp_file.path, 'wb') do |file|
            HTTPClient.new.post(@url, [],
              {'Content-Type' => @content_type, :data => file})
          end
        else
          raise IncompatibleOutputError, "Don't know to store output URL: #{@url}"
      end
    end
    
    def close
      @temp_file.unlink
    end
    
{'fraction_non_alphanumeric': 0.11280814576634512, 'fraction_numerical': 0.007502679528403001, 'mean_word_length': 1.9255485893416928, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3671207', 'n_tokens_mistral': 1128, 'n_tokens_neox': 1121, 'n_words': 247}