<commit_before># encoding: utf-8
require 'fileutils'
require 'open3'
require 'uri'
require_relative './exceptions'
require_relative './source_file'
require_relative './url_translator/osm'
require_relative './url_translator/osm2'
require_relative './url_translator/fusion_tables'
require_relative './url_translator/github'
require_relative './url_translator/google_maps'
require_relative './url_translator/google_docs'
require_relative './url_translator/kimono_labs'
require_relative './unp'
require_relative '../../../../lib/carto/http/client'
require_relative '../../../../lib/carto/url_validator'
require_relative '../helpers/quota_check_helpers.rb'

module CartoDB
  module Importer2
    class Downloader
      include CartoDB::Importer2::QuotaCheckHelpers
      extend Carto::UrlValidator

      def self.supported_extensions
        @supported_extensions ||= CartoDB::Importer2::Unp::SUPPORTED_FORMATS
                                  .concat(CartoDB::Importer2::Unp::COMPRESSED_EXTENSIONS)
                                  .sort_by(&:length).reverse
      end

      def self.supported_extensions_match
        @supported_extensions_match ||= supported_extensions.map { |ext|
          ext = ext.gsub('.', '\\.')
          [/#{ext}$/i, /#{ext}(?=\.)/i, /#{ext}(?=\?)/i, /#{ext}(?=&)/i]
        }.flatten
      end

      def self.url_filename_regex
        return @url_filename_regex if @url_filename_regex

        se_match_regex = Regexp.union(supported_extensions_match)
        @url_filename_regex = Regexp.new("[[:word:]-]+#{se_match_regex}+", Regexp::IGNORECASE)
      end

      def provides_stream?
        false
      end

      def http_download?
        true
      end

      def multi_resource_import_supported?
        false
      end

      attr_reader :source_file, :etag, :last_modified, :http_response_code, :datasource

      def initialize(url, http_options = {}, options = {})
        raise UploadError if url.nil?

        @http_options = http_options
        @options = options
        @downloaded_bytes = 0
        @translated_url = translate_url(url)

        user_id = options[:user_id]
        @user = Carto::User.find(user_id) if user_id

        @downloaded_bytes = 0
      end

      def run(available_quota_in_bytes = nil)
        if @translated_url =~ %r{://}
          if available_quota_in_bytes
            raise_if_over_storage_quota(requested_quota: content_length,
                                        available_quota: available_quota_in_bytes.to_i,
                                        user_id: @user.try(:id))
          end

          modified? ? download_and_store : @source_file = nil
        else
          @source_file = SourceFile.new(@translated_url)
        end

        self
      end

      def modified?
        previous_etag = @http_options.fetch(:etag, false)
        previous_last_modified = @http_options.fetch(:last_modified, false)

        return true unless previous_etag || previous_last_modified

        etag_changed = etag && previous_etag && etag != previous_etag
        last_modified_changed = (last_modified &&
                                 previous_last_modified &&
                                 previous_last_modified.to_i < last_modified.to_i)

        etag_changed || last_modified_changed
      end

      private

      URL_ESCAPED_CHARACTERS = 'áéíóúÁÉÍÓÚñÑçÇàèìòùÀÈÌÒÙ'.freeze

      def translate_url(url)
        translator = supported_translator(url)

        raw_url = if translator
                    if translator.respond_to?(:rename_destination)
                      @custom_filename = translator.rename_destination(url)
                    end

                    translator.translate(url)
                  else
                    url
                  end

        raw_url.try(:is_a?, String) ? URI.escape(raw_url.strip, URL_ESCAPED_CHARACTERS) : raw_url
      end

      def headers
        return @headers if @headers

        response = http_client.head(@translated_url, typhoeus_options)

        @http_response_code = response.code
        response.success? ? process_headers(response.headers) : raise_error_for_response(response)

        @headers
      end

      def process_headers(headers)
        @headers = headers

        CartoDB::Importer2::Downloader.validate_url!(response.effective_url)

        basename = @custom_filename ||
                   filename_from_headers ||
                   filename_from_url ||
                   SecureRandom.urlsafe_base64

        @filename = name_with_extension(basename)
        @etag = etag
        @last_modified = last_modified
      end

      MAX_REDIRECTS = 5
      HTTP_CONNECT_TIMEOUT_SECONDS = 60
      DEFAULT_HTTP_REQUEST_TIMEOUT_SECONDS = 600

      def typhoeus_options
        verify_ssl = @http_options.fetch(:verify_ssl_cert, false)
        cookiejar = Tempfile.new('cookiejar_').path

        {
          cookiefile:       cookiejar,
          cookiejar:        cookiejar,
          followlocation:   true,
          ssl_verifypeer:   verify_ssl,
          ssl_verifyhost:   (verify_ssl ? 2 : 0),
          forbid_reuse:     true,
          connecttimeout:   HTTP_CONNECT_TIMEOUT_SECONDS,
          timeout:          @http_options.fetch(:http_timeout, DEFAULT_HTTP_REQUEST_TIMEOUT_SECONDS),
          maxredirs:        MAX_REDIRECTS
        }
      end

      FILENAME_PREFIX = 'importer_'.freeze

      def download_and_store
        file = Tempfile.new(FILENAME_PREFIX, encoding: 'ascii-8bit')
        binded_request(@translated_url, file, size_limit_in_bytes: @user.try(:max_import_file_size)).run

        file_path = if @filename
                      new_file_path = File.join(Pathname.new(file.path).dirname, @filename)
                      File.rename(file.path, new_file_path)

                      new_file_path
                    else
                      file.path
                    end

        @source_file = SourceFile.new(file_path, @filename)
      ensure
        file.close
        file.unlink
      end

      def binded_request(url, file, size_limit_in_bytes: 5_368_709_120)
        request = Typhoeus::Request.new(url, typhoeus_options)

        request.on_headers do |response|
          @http_response_code = response.code

          response.success? ? process_headers(response.headers) : raise_error_for_response(response)
        end

        request.on_body do |chunk|
          if (@downloaded_bytes += chunk.bytesize) > size_limit_in_bytes
            raise FileTooBigError.new("download file too big (> #{MAX_DOWNLOAD_SIZE} bytes)")
          else
            file.write(chunk)
          end
        end

        request.on_complete do |response|
          raise_error_for_response(response) unless response.success?
        end

        request<commit_msg>validate url outside<commit_after># encoding: utf-8
require 'fileutils'
require 'open3'
require 'uri'
require_relative './exceptions'
require_relative './source_file'
require_relative './url_translator/osm'
require_relative './url_translator/osm2'
require_relative './url_translator/fusion_tables'
require_relative './url_translator/github'
require_relative './url_translator/google_maps'
require_relative './url_translator/google_docs'
require_relative './url_translator/kimono_labs'
require_relative './unp'
require_relative '../../../../lib/carto/http/client'
require_relative '../../../../lib/carto/url_validator'
require_relative '../helpers/quota_check_helpers.rb'

module CartoDB
  module Importer2
    class Downloader
      include CartoDB::Importer2::QuotaCheckHelpers
      extend Carto::UrlValidator

      def self.supported_extensions
        @supported_extensions ||= CartoDB::Importer2::Unp::SUPPORTED_FORMATS
                                  .concat(CartoDB::Importer2::Unp::COMPRESSED_EXTENSIONS)
                                  .sort_by(&:length).reverse
      end

      def self.supported_extensions_match
        @supported_extensions_match ||= supported_extensions.map { |ext|
          ext = ext.gsub('.', '\\.')
          [/#{ext}$/i, /#{ext}(?=\.)/i, /#{ext}(?=\?)/i, /#{ext}(?=&)/i]
        }.flatten
      end

      def self.url_filename_regex
        return @url_filename_regex if @url_filename_regex

        se_match_regex = Regexp.union(supported_extensions_match)
        @url_filename_regex = Regexp.new("[[:word:]-]+#{se_match_regex}+", Regexp::IGNORECASE)
      end

      def provides_stream?
        false
      end

      def http_download?
        true
      end

      def multi_resource_import_supported?
        false
      end

      attr_reader :source_file, :etag, :last_modified, :http_response_code, :datasource

      def initialize(url, http_options = {}, options = {})
        raise UploadError if url.nil?

        @http_options = http_options
        @options = options
        @downloaded_bytes = 0
        @translated_url = translate_url(url)

        user_id = options[:user_id]
        @user = Carto::User.find(user_id) if user_id

        @downloaded_bytes = 0
      end

      def run(available_quota_in_bytes = nil)
        if @translated_url =~ %r{://}
          if available_quota_in_bytes
            raise_if_over_storage_quota(requested_quota: content_length,
                                        available_quota: available_quota_in_bytes.to_i,
                                        user_id: @user.try(:id))
          end

          modified? ? download_and_store : @source_file = nil
        else
          @source_file = SourceFile.new(@translated_url)
        end

        self
      end

      def modified?
        previous_etag = @http_options.fetch(:etag, false)
        previous_last_modified = @http_options.fetch(:last_modified, false)

        return true unless previous_etag || previous_last_modified

        etag_changed = etag && previous_etag && etag != previous_etag
        last_modified_changed = (last_modified &&
                                 previous_last_modified &&
                                 previous_last_modified.to_i < last_modified.to_i)

        etag_changed || last_modified_changed
      end

      private

      URL_ESCAPED_CHARACTERS = 'áéíóúÁÉÍÓÚñÑçÇàèìòùÀÈÌÒÙ'.freeze

      def translate_url(url)
        translator = supported_translator(url)

        raw_url = if translator
                    if translator.respond_to?(:rename_destination)
                      @custom_filename = translator.rename_destination(url)
                    end

                    translator.translate(url)
                  else
                    url
                  end

        raw_url.try(:is_a?, String) ? URI.escape(raw_url.strip, URL_ESCAPED_CHARACTERS) : raw_url
      end

      def headers
        return @headers if @headers

        response = http_client.head(@translated_url, typhoeus_options)

        @http_response_code = response.code
        CartoDB::Importer2::Downloader.validate_url!(response.effective_url)
        response.success? ? process_headers(response.headers) : raise_error_for_response(response)

        @headers
      end

      def process_headers(headers)
        @headers = headers

        basename = @custom_filename ||
                   filename_from_headers ||
                   filename_from_url ||
                   SecureRandom.urlsafe_base64

        @filename = name_with_extension(basename)
        @etag = etag
        @last_modified = last_modified
      end

      MAX_REDIRECTS = 5
      HTTP_CONNECT_TIMEOUT_SECONDS = 60
      DEFAULT_HTTP_REQUEST_TIMEOUT_SECONDS = 600

      def typhoeus_options
        verify_ssl = @http_options.fetch(:verify_ssl_cert, false)
        cookiejar = Tempfile.new('cookiejar_').path

        {
          cookiefile:       cookiejar,
          cookiejar:        cookiejar,
          followlocation:   true,
          ssl_verifypeer:   verify_ssl,
          ssl_verifyhost:   (verify_ssl ? 2 : 0),
          forbid_reuse:     true,
          connecttimeout:   HTTP_CONNECT_TIMEOUT_SECONDS,
          timeout:          @http_options.fetch(:http_timeout, DEFAULT_HTTP_REQUEST_TIMEOUT_SECONDS),
          maxredirs:        MAX_REDIRECTS
        }
      end

      FILENAME_PREFIX = 'importer_'.freeze

      def download_and_store
        file = Tempfile.new(FILENAME_PREFIX, encoding: 'ascii-8bit')
        binded_request(@translated_url, file, size_limit_in_bytes: @user.try(:max_import_file_size)).run

        file_path = if @filename
                      new_file_path = File.join(Pathname.new(file.path).dirname, @filename)
                      File.rename(file.path, new_file_path)

                      new_file_path
                    else
                      file.path
                    end

        @source_file = SourceFile.new(file_path, @filename)
      ensure
        file.close
        file.unlink
      end

      def binded_request(url, file, size_limit_in_bytes: 5_368_709_120)
        request = Typhoeus::Request.new(url, typhoeus_options)

        request.on_headers do |response|
          @http_response_code = response.code
          CartoDB::Importer2::Downloader.validate_url!(response.effective_url)

          response.success? ? process_headers(response.headers) : raise_error_for_response(response)
        end

        request.on_body do |chunk|
          if (@downloaded_bytes += chunk.bytesize) > size_limit_in_bytes
            raise FileTooBigError.new("download file too big (> #{MAX_DOWNLOAD_SIZE} bytes)")
          else
            file.write(chunk)
          end
        end

        request.on_complete do |response|
          raise_error_for_response(response) unless response.success?
        end

        request
{'fraction_non_alphanumeric': 0.08722764287286333, 'fraction_numerical': 0.004621817915046585, 'mean_word_length': 2.1739231664726426, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 31, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '243337', 'n_tokens_mistral': 4073, 'n_tokens_neox': 3909, 'n_words': 678}