<commit_before># encoding: utf-8
require 'fileutils'
require 'open3'
require 'uri'
require_relative './exceptions'
require_relative './source_file'
require_relative './url_translator/osm'
require_relative './url_translator/osm2'
require_relative './url_translator/fusion_tables'
require_relative './url_translator/github'
require_relative './url_translator/google_maps'
require_relative './url_translator/google_docs'
require_relative './url_translator/kimono_labs'
require_relative './unp'
require_relative '../../../../lib/carto/http/client'
require_relative '../../../../lib/carto/url_validator'
require_relative '../helpers/quota_check_helpers.rb'

module CartoDB
  module Importer2
    class Downloader
      include CartoDB::Importer2::QuotaCheckHelpers, Carto::UrlValidator

      def self.supported_extensions
        @supported_extensions ||= CartoDB::Importer2::Unp::SUPPORTED_FORMATS
                                  .concat(CartoDB::Importer2::Unp::COMPRESSED_EXTENSIONS)
                                  .sort_by(&:length).reverse
      end

      def self.supported_extensions_match
        @supported_extensions_match ||= supported_extensions.map { |ext|
          ext = ext.gsub('.', '\\.')
          [/#{ext}$/i, /#{ext}(?=\.)/i, /#{ext}(?=\?)/i, /#{ext}(?=&)/i]
        }.flatten
      end

      def self.url_filename_regex
        return @url_filename_regex if @url_filename_regex

        se_match_regex = Regexp.union(supported_extensions_match)
        @url_filename_regex = Regexp.new("[[:word:]-]+#{se_match_regex}+", Regexp::IGNORECASE)
      end

      def provides_stream?
        false
      end

      def http_download?
        true
      end

      def multi_resource_import_supported?
        false
      end

      attr_reader :source_file, :etag, :last_modified, :http_response_code, :datasource

      def initialize(user_id, url, http_options = {}, options = {})
        raise UploadError if url.nil?

        @http_options = http_options
        @options = options
        @downloaded_bytes = 0
        @translated_url = translate_url(url)

        @user = Carto::User.find(user_id) if user_id

        @downloaded_bytes = 0
      end

      def run(available_quota_in_bytes = nil)
        if @translated_url =~ %r{://}
          if available_quota_in_bytes
            raise_if_over_storage_quota(requested_quota: content_length,
                                        available_quota: available_quota_in_bytes.to_i,
                                        user_id: @user.try(:id))
          end

          modified? ? download_and_store : @source_file = nil
        else
          @source_file = SourceFile.new(@translated_url)
        end

        self
      end

      def modified?
        previous_etag = @http_options.fetch(:etag, false)
        previous_last_modified = @http_options.fetch(:last_modified, false)

        return true unless previous_etag || previous_last_modified<commit_msg>fail early if user_id nil<commit_after># encoding: utf-8
require 'fileutils'
require 'open3'
require 'uri'
require_relative './exceptions'
require_relative './source_file'
require_relative './url_translator/osm'
require_relative './url_translator/osm2'
require_relative './url_translator/fusion_tables'
require_relative './url_translator/github'
require_relative './url_translator/google_maps'
require_relative './url_translator/google_docs'
require_relative './url_translator/kimono_labs'
require_relative './unp'
require_relative '../../../../lib/carto/http/client'
require_relative '../../../../lib/carto/url_validator'
require_relative '../helpers/quota_check_helpers.rb'

module CartoDB
  module Importer2
    class Downloader
      include CartoDB::Importer2::QuotaCheckHelpers, Carto::UrlValidator

      def self.supported_extensions
        @supported_extensions ||= CartoDB::Importer2::Unp::SUPPORTED_FORMATS
                                  .concat(CartoDB::Importer2::Unp::COMPRESSED_EXTENSIONS)
                                  .sort_by(&:length).reverse
      end

      def self.supported_extensions_match
        @supported_extensions_match ||= supported_extensions.map { |ext|
          ext = ext.gsub('.', '\\.')
          [/#{ext}$/i, /#{ext}(?=\.)/i, /#{ext}(?=\?)/i, /#{ext}(?=&)/i]
        }.flatten
      end

      def self.url_filename_regex
        return @url_filename_regex if @url_filename_regex

        se_match_regex = Regexp.union(supported_extensions_match)
        @url_filename_regex = Regexp.new("[[:word:]-]+#{se_match_regex}+", Regexp::IGNORECASE)
      end

      def provides_stream?
        false
      end

      def http_download?
        true
      end

      def multi_resource_import_supported?
        false
      end

      attr_reader :source_file, :etag, :last_modified, :http_response_code, :datasource

      def initialize(user_id, url, http_options = {}, options = {})
        raise UploadError unless user_id && url

        @user = Carto::User.find(user_id) if user_id
        @translated_url = translate_url(url)
        @http_options = http_options
        @options = options

        @downloaded_bytes = 0
      end

      def run(available_quota_in_bytes = nil)
        if @translated_url =~ %r{://}
          if available_quota_in_bytes
            raise_if_over_storage_quota(requested_quota: content_length,
                                        available_quota: available_quota_in_bytes.to_i,
                                        user_id: @user.try(:id))
          end

          modified? ? download_and_store : @source_file = nil
        else
          @source_file = SourceFile.new(@translated_url)
        end

        self
      end

      def modified?
        previous_etag = @http_options.fetch(:etag, false)
        previous_last_modified = @http_options.fetch(:last_modified, false)

        return true unless previous_etag || previous_last_modified
{'fraction_non_alphanumeric': 0.11033543333900903, 'fraction_numerical': 0.0028946024178443724, 'mean_word_length': 2.5904645476772616, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2548897', 'n_tokens_mistral': 1777, 'n_tokens_neox': 1744, 'n_words': 314}