<commit_before># encoding: utf-8
require 'typhoeus'
require_relative './exceptions'
require_relative './source_file'
require_relative '../../../data-repository/filesystem/local'
require_relative './url_translator/osm'
require_relative './url_translator/fusion_tables'
require_relative './url_translator/github'
require_relative './url_translator/google_maps'

module CartoDB
  module Importer2
    class Downloader

      DEFAULT_FILENAME        = 'importer'
      CONTENT_DISPOSITION_RE  = %r{attachment; filename=(.*;|.*)}
      URL_RE                  = %r{://}
      URL_TRANSLATORS         = [
                                  UrlTranslator::OSM,
                                  UrlTranslator::FusionTables,
                                  UrlTranslator::GitHub,
                                  UrlTranslator::GoogleMaps
                                ]

      def initialize(url, seed=nil, repository=nil)
        self.url          = url
        raise UploadError if url.nil?
        self.seed         = seed
        self.repository   = repository || 
                            DataRepository::Filesystem::Local.new(temporary_directory)
      end #initialize

      def run
        unless url =~ URL_RE
          self.source_file = SourceFile.new(url)
          return self
        end

        response          = Typhoeus.get(translate(url), followlocation: true)
        name              = name_from(response.headers, url)
        self.source_file  = SourceFile.new(filepath(name), name)

        repository.store(source_file.path, StringIO.new(response.response_body))
        self
      end #run

      def name_from(headers, url)
        name_from_http(headers) || name_in(url)
      end #filename_from

      attr_reader :source_file
      attr_reader :url

      private
      
      attr_accessor :repository, :seed
      attr_writer   :url, :source_file

      def translators
        URL_TRANSLATORS.map(&:new)
      end #translators

      def translate(url)
        translator = translators.find { |translator| translator.supported?(url) }
        return url unless translator
        translator.translate(url)
      end #translated_url

      def filename
        [DEFAULT_FILENAME, seed].compact.join('_')
      end #filename

      def filepath(name=nil)
        repository.fullpath_for(name || filename)
      end #filepath

      def name_from_http(headers)
        disposition = headers.fetch('Content-Disposition', nil)
        return false unless disposition
        filename = disposition.match(CONTENT_DISPOSITION_RE).to_a[1]
        return false unless filename
        filename.delete("'").delete('"').delete(';')
      end #name_from_http

      def name_in(url)
        url.split('/').last.split('?').first
      end #name_in

      def temporary_directory
        return @temporary_directory if @temporary_directory
        tempfile              = Tempfile.new("")
        @temporary_directory  = tempfile.path

        tempfile.close!
        Dir.mkdir(temporary_directory)
        @temporary_directory
      end #temporary_directory
    end # Downloader
  end # Importer2
end # CartoDB

<commit_msg>[importer] Workaround for eager content-disposition regex<commit_after># encoding: utf-8
require 'typhoeus'
require_relative './exceptions'
require_relative './source_file'
require_relative '../../../data-repository/filesystem/local'
require_relative './url_translator/osm'
require_relative './url_translator/fusion_tables'
require_relative './url_translator/github'
require_relative './url_translator/google_maps'

module CartoDB
  module Importer2
    class Downloader

      DEFAULT_FILENAME        = 'importer'
      CONTENT_DISPOSITION_RE  = %r{attachment; filename=(.*;|.*)}
      URL_RE                  = %r{://}
      URL_TRANSLATORS         = [
                                  UrlTranslator::OSM,
                                  UrlTranslator::FusionTables,
                                  UrlTranslator::GitHub,
                                  UrlTranslator::GoogleMaps
                                ]

      def initialize(url, seed=nil, repository=nil)
        self.url          = url
        raise UploadError if url.nil?
        self.seed         = seed
        self.repository   = repository || 
                            DataRepository::Filesystem::Local.new(temporary_directory)
      end #initialize

      def run
        unless url =~ URL_RE
          self.source_file = SourceFile.new(url)
          return self
        end

        response          = Typhoeus.get(translate(url), followlocation: true)
        name              = name_from(response.headers, url)
        self.source_file  = SourceFile.new(filepath(name), name)

        repository.store(source_file.path, StringIO.new(response.response_body))
        self
      end #run

      def name_from(headers, url)
        name_from_http(headers) || name_in(url)
      end #filename_from

      attr_reader :source_file
      attr_reader :url

      private
      
      attr_accessor :repository, :seed
      attr_writer   :url, :source_file

      def translators
        URL_TRANSLATORS.map(&:new)
      end #translators

      def translate(url)
        translator = translators.find { |translator| translator.supported?(url) }
        return url unless translator
        translator.translate(url)
      end #translated_url

      def filename
        [DEFAULT_FILENAME, seed].compact.join('_')
      end #filename

      def filepath(name=nil)
        repository.fullpath_for(name || filename)
      end #filepath

      def name_from_http(headers)
        disposition = headers.fetch('Content-Disposition', nil)
        return false unless disposition
        filename = disposition.match(CONTENT_DISPOSITION_RE).to_a[1]
        return false unless filename
        filename.delete("'").delete('"').split(';').first
      end #name_from_http

      def name_in(url)
        url.split('/').last.split('?').first
      end #name_in

      def temporary_directory
        return @temporary_directory if @temporary_directory
        tempfile              = Tempfile.new("")
        @temporary_directory  = tempfile.path

        tempfile.close!
        Dir.mkdir(temporary_directory)
        @temporary_directory
      end #temporary_directory
    end # Downloader
  end # Importer2
end # CartoDB


{'fraction_non_alphanumeric': 0.08989118435577985, 'fraction_numerical': 0.0012616306576249802, 'mean_word_length': 2.072674418604651, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6950636', 'n_tokens_mistral': 1800, 'n_tokens_neox': 1721, 'n_words': 347}