<commit_before># encoding: utf-8
require 'fileutils'
require 'typhoeus'
require 'open3'
require_relative './exceptions'
require_relative './source_file'
require_relative '../../../data-repository/filesystem/local'
require_relative './url_translator/osm'
require_relative './url_translator/osm2'
require_relative './url_translator/fusion_tables'
require_relative './url_translator/github'
require_relative './url_translator/google_maps'
require_relative './url_translator/google_docs'
require_relative './url_translator/kimono_labs'
require_relative './unp'

module CartoDB
  module Importer2
    class Downloader

      DEFAULT_FILENAME        = 'importer'
      CONTENT_DISPOSITION_RE  = %r{;\s*filename=(.*;|.*)}
      URL_RE                  = %r{://}
      URL_TRANSLATORS         = [
                                  UrlTranslator::OSM2,
                                  UrlTranslator::OSM,
                                  UrlTranslator::FusionTables,
                                  UrlTranslator::GitHub,
                                  UrlTranslator::GoogleMaps,
                                  UrlTranslator::GoogleDocs,
                                  UrlTranslator::KimonoLabs
                                ]

      def initialize(url, http_options={}, seed=nil, repository=nil)
        @url          = url
        raise UploadError if url.nil?

        @http_options = http_options
        @seed         = seed
        @repository   = repository || DataRepository::Filesystem::Local.new(temporary_directory)
        @datasource

        translators = URL_TRANSLATORS.map(&:new)
        translator = translators.find { |translator| translator.supported?(url) }
        if translator.nil?
          @translated_url = url
          @custom_filename = nil
        else
          @translated_url = translator.translate(url)
          @custom_filename = translator.respond_to?(:rename_destination) ? translator.rename_destination(url) : nil
        end
      end

      def provides_stream?
        false
      end

      def run(available_quota_in_bytes=nil)
        set_local_source_file || set_downloaded_source_file(available_quota_in_bytes)
        self
      end

      def set_local_source_file
        return false if valid_url?
        self.source_file = SourceFile.new(url)
        self
      end

      def valid_url?
        url =~ URL_RE
      end

      def set_downloaded_source_file(available_quota_in_bytes=nil)
        raise_if_over_storage_quota(headers, available_quota_in_bytes)
        @etag           = etag_from(headers)
        @last_modified  = last_modified_from(headers)
        return self unless modified?

        download_and_store

        self.source_file  = nil unless modified?
        self
      end


      def raise_if_over_storage_quota(headers, available_quota_in_bytes=nil)
        return self unless available_quota_in_bytes
        raise StorageQuotaExceededError if 
          content_length_from(headers) > available_quota_in_bytes.to_i
      end

      def typhoeus_options
        verify_ssl = http_options.fetch(:verify_ssl_cert, false)
        {
          cookiefile:     cookiejar,
          cookiejar:      cookiejar,
          followlocation: true,
          ssl_verifypeer: verify_ssl,
          ssl_verifyhost: (verify_ssl ? 2 : 0)
        }
      end 

      def headers
        @headers ||= Typhoeus.head(@translated_url, typhoeus_options).headers
      end

      def cookiejar
        repository.fullpath_for("#{seed}_cookiejar")
      end

      def download_and_store
        name = ''
        download_error = false

        temp_name = filepath(DEFAULT_FILENAME << '_' << random_name)

        downloaded_file = File.open(temp_name, 'wb')
        request = Typhoeus::Request.new(@translated_url, typhoeus_options)
        request.on_headers do |response|
          unless response.success?
            download_error = true
          end
        end
        request.on_body do |chunk|
          downloaded_file.write(chunk)
        end
        request.on_complete do |response|
          downloaded_file.close

          headers = response.headers

          name            = name_from(headers, url, @custom_filename)
          @etag           = etag_from(headers)
          @last_modified  = last_modified_from(headers)
        end
        request.run

        raise DownloadError if download_error

        File.rename(temp_name, filepath(name))<commit_msg>CDB-4368<commit_after># encoding: utf-8
require 'fileutils'
require 'typhoeus'
require 'open3'
require_relative './exceptions'
require_relative './source_file'
require_relative '../../../data-repository/filesystem/local'
require_relative './url_translator/osm'
require_relative './url_translator/osm2'
require_relative './url_translator/fusion_tables'
require_relative './url_translator/github'
require_relative './url_translator/google_maps'
require_relative './url_translator/google_docs'
require_relative './url_translator/kimono_labs'
require_relative './unp'

module CartoDB
  module Importer2
    class Downloader

      DEFAULT_FILENAME        = 'importer'
      CONTENT_DISPOSITION_RE  = %r{;\s*filename=(.*;|.*)}
      URL_RE                  = %r{://}
      URL_TRANSLATORS         = [
                                  UrlTranslator::OSM2,
                                  UrlTranslator::OSM,
                                  UrlTranslator::FusionTables,
                                  UrlTranslator::GitHub,
                                  UrlTranslator::GoogleMaps,
                                  UrlTranslator::GoogleDocs,
                                  UrlTranslator::KimonoLabs
                                ]

      def initialize(url, http_options={}, seed=nil, repository=nil)
        @url          = url
        raise UploadError if url.nil?

        @http_options = http_options
        @seed         = seed
        @repository   = repository || DataRepository::Filesystem::Local.new(temporary_directory)
        @datasource

        translators = URL_TRANSLATORS.map(&:new)
        translator = translators.find { |translator| translator.supported?(url) }
        if translator.nil?
          @translated_url = url
          @custom_filename = nil
        else
          @translated_url = translator.translate(url)
          @custom_filename = translator.respond_to?(:rename_destination) ? translator.rename_destination(url) : nil
        end
      end

      def provides_stream?
        false
      end

      def run(available_quota_in_bytes=nil)
        set_local_source_file || set_downloaded_source_file(available_quota_in_bytes)
        self
      end

      def set_local_source_file
        return false if valid_url?
        self.source_file = SourceFile.new(url)
        self
      end

      def valid_url?
        url =~ URL_RE
      end

      def set_downloaded_source_file(available_quota_in_bytes=nil)
        raise_if_over_storage_quota(headers, available_quota_in_bytes)
        @etag           = etag_from(headers)
        @last_modified  = last_modified_from(headers)
        return self unless modified?

        download_and_store

        self.source_file  = nil unless modified?
        self
      end


      def raise_if_over_storage_quota(headers, available_quota_in_bytes=nil)
        return self unless available_quota_in_bytes
        raise StorageQuotaExceededError if 
          content_length_from(headers) > available_quota_in_bytes.to_i
      end

      def typhoeus_options
        verify_ssl = http_options.fetch(:verify_ssl_cert, false)
        {
          cookiefile:     cookiejar,
          cookiejar:      cookiejar,
          followlocation: true,
          ssl_verifypeer: verify_ssl,
          ssl_verifyhost: (verify_ssl ? 2 : 0)
        }
      end 

      def headers
        @headers ||= Typhoeus.head(@translated_url, typhoeus_options).headers
      end

      def cookiejar
        repository.fullpath_for("#{seed}_cookiejar")
      end

      def download_and_store
        name = ''
        download_error = false
        error_response = nil

        temp_name = filepath(DEFAULT_FILENAME << '_' << random_name)

        downloaded_file = File.open(temp_name, 'wb')
        request = Typhoeus::Request.new(@translated_url, typhoeus_options)
        request.on_headers do |response|
          unless response.success?
            download_error = true
            error_response = response
          end
        end
        request.on_body do |chunk|
          downloaded_file.write(chunk)
        end
        request.on_complete do |response|
          downloaded_file.close

          headers = response.headers

          name            = name_from(headers, url, @custom_filename)
          @etag           = etag_from(headers)
          @last_modified  = last_modified_from(headers)
        end
        request.run

        if download_error && !error_response.nil?
          raise DownloadError.new("DOWNLOAD ERROR: Code:#{error_response.code} Body:#{error_response.body}")
        end

        File.rename(temp_name, filepath(name))
{'fraction_non_alphanumeric': 0.07759285793012234, 'fraction_numerical': 0.001983908299349719, 'mean_word_length': 2.063470627954085, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '815131', 'n_tokens_mistral': 2649, 'n_tokens_neox': 2515, 'n_words': 460}