<commit_before># encoding: utf-8
require 'tempfile'
require 'fileutils'
require 'iconv'
require 'open3'
require_relative '../importer'
require_relative './exceptions'

module CartoDB
  module Importer2
    class Unp
      HIDDEN_FILE_REGEX     = /^(\.|\_{2})/
      UNP_READ_ERROR_REGEX  = /.*Cannot read.*/
      COMPRESSED_EXTENSIONS = %w{ .zip .gz .tgz .tar.gz .bz2 .tar .kmz }
      SUPPORTED_FORMATS       = %w{
        .csv .shp .ods .xls .xlsx .tif .tiff .kml .kmz
        .txt .js .json .tar .gz .tgz .osm .bz2 .geojson 
        .gpx .sql .tab
      }

      attr_reader :source_files, :temporary_directory

      def initialize
        @source_files = []
      end #initialize

      def run(path)
        return without_unpacking(path) unless compressed?(path)
        extract(path)
        crawl(temporary_directory).each { |path| process(path) }

        self
      rescue => exception
        raise ExtractionError
      end #run

      def without_unpacking(path)
        self.source_files.push(source_file_for(normalize(path)))
        self
      end #without_unpacking

      def compressed?(path)
        COMPRESSED_EXTENSIONS.include?(File.extname(path))
      end #compressed?

      def process(path)
        source_files.push(source_file_for(path)) if supported?(path)
      end #process

      def crawl(path, files=[])
        Dir.foreach(path) do |subpath|
          next if hidden?(subpath)

          fullpath = normalize("#{path}/#{subpath}")
          (crawl(fullpath, files) and next) if File.directory?(fullpath)
          files.push(fullpath)
        end # foreach

        files
      end #crawl

      def extract(path)
        generate_temporary_directory
        current_directory   = Dir.pwd

        Dir.chdir(temporary_directory)
        stdout, stderr, status  = Open3.capture3(command_for(path))
        Dir.chdir(current_directory)

        raise ExtractionError if unp_failure?(stdout + stderr, status)
        self
      end #extract

      def source_file_for(path)
        SourceFile.new(path)
      end #source_file_for

      def command_for(path)
        "`which unp` #{path}"
      end #command_for

     def supported?(filename)
        SUPPORTED_FORMATS.include?(File.extname(filename))
      end #supported?

      def normalize(filename)
        normalized = underscore(filename)
        rename(filename, normalized)
        normalized
      end #normalize

      def underscore(filename)
        Iconv.new('UTF-8//IGNORE', 'UTF-8').iconv(filename)
          .gsub(' ', '_')
          .downcase<commit_msg>[importer] Sanitize filepath before extracting<commit_after># encoding: utf-8
require 'tempfile'
require 'fileutils'
require 'iconv'
require 'open3'
require_relative '../importer'
require_relative './exceptions'

module CartoDB
  module Importer2
    class Unp
      HIDDEN_FILE_REGEX     = /^(\.|\_{2})/
      UNP_READ_ERROR_REGEX  = /.*Cannot read.*/
      COMPRESSED_EXTENSIONS = %w{ .zip .gz .tgz .tar.gz .bz2 .tar .kmz }
      SUPPORTED_FORMATS       = %w{
        .csv .shp .ods .xls .xlsx .tif .tiff .kml .kmz
        .txt .js .json .tar .gz .tgz .osm .bz2 .geojson 
        .gpx .sql .tab
      }

      attr_reader :source_files, :temporary_directory

      def initialize
        @source_files = []
      end #initialize

      def run(path)
        return without_unpacking(path) unless compressed?(path)
        extract(normalize(path))
        crawl(temporary_directory).each { |path| process(path) }
        self
      rescue => exception
        raise ExtractionError
      end #run

      def without_unpacking(path)
        self.source_files.push(source_file_for(normalize(path)))
        self
      end #without_unpacking

      def compressed?(path)
        COMPRESSED_EXTENSIONS.include?(File.extname(path))
      end #compressed?

      def process(path)
        source_files.push(source_file_for(path)) if supported?(path)
      end #process

      def crawl(path, files=[])
        Dir.foreach(path) do |subpath|
          next if hidden?(subpath)

          fullpath = normalize("#{path}/#{subpath}")
          (crawl(fullpath, files) and next) if File.directory?(fullpath)
          files.push(fullpath)
        end # foreach

        files
      end #crawl

      def extract(path)
        generate_temporary_directory
        current_directory   = Dir.pwd

        Dir.chdir(temporary_directory)
        stdout, stderr, status  = Open3.capture3(command_for(path))
        Dir.chdir(current_directory)

        raise ExtractionError if unp_failure?(stdout + stderr, status)
        self
      end #extract

      def source_file_for(path)
        SourceFile.new(path)
      end #source_file_for

      def command_for(path)
        "`which unp` #{path}"
      end #command_for

     def supported?(filename)
        SUPPORTED_FORMATS.include?(File.extname(filename))
      end #supported?

      def normalize(filename)
        normalized = underscore(filename)
        rename(filename, normalized)
        normalized
      end #normalize

      def underscore(filename)
        Iconv.new('UTF-8//IGNORE', 'UTF-8').iconv(filename)
          .gsub(' ', '_')
          .gsub(/\(/, '')
          .gsub(/\)/, '')
          .downcase
{'fraction_non_alphanumeric': 0.1078074045655093, 'fraction_numerical': 0.0038365624400537117, 'mean_word_length': 2.25875, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1197433', 'n_tokens_mistral': 1683, 'n_tokens_neox': 1622, 'n_words': 359}