<commit_before># coding: UTF-8

require 'spec_helper'

describe CartoDB::Importer do
  homepath = File.expand_path('~')
  if File.exists? "#{homepath}/Dropbox/ec2-keys/.amazon_key"
    require 'aws-sdk'
    
    local_storage_dir = "/tmp/failed_files"
    unless File.directory? local_storage_dir
      Dir.mkdir(local_storage_dir)
    end
    
    s3 = AWS::S3.new({
      :access_key_id => File.open("#{homepath}/Dropbox/ec2-keys/.amazon_key", "rb").read.strip,
      :secret_access_key => File.open("#{homepath}/Dropbox/ec2-keys/.amazon_secret", "rb").read.strip
    })
    bucket = s3.buckets[:failed_imports]
    
    it "test failed files from S3" do
      bucket.objects.each do |remote_object|
        debugger
        #key = "temp/test_1333394922.csv"
        key = object.key
        name = key.split('/').last
        local_path = "/#{local_storage_dir}/#{name}"
        #remote_object = bucket.objects[key]
        remote_object.write(File.open(local_path,'w+'))
        if 0 == %x{wc -l #{local_path}}.split.first.to_i
          File.delete(local_path)
          #remote_object.delete
          warn 'empty file, skipping'
        else
          importer = create_importer local_path, 'import_file'
          result   = importer.import!
          result.name.should          == 'import_file'
        end
      end
    end
  else 
    warn "You don't have Vizzuality S3 keys available."
    warn "If you aren't from Vizzuality, this is normal."
  end
  ##################################################
  # configuration & helpers for tests
  ##################################################
  before(:all) do
    @db = CartoDB::ImportDatabaseConnection.connection
    @db_opts = {:database => "cartodb_importer_test", 
                :username => "postgres", :password => '',
                :host => 'localhost', 
                :port => 5432}
  end
  
  after(:all) do
    CartoDB::ImportDatabaseConnection.drop
  end
  def file file
    # File.expand_path("../../support/data/#{file}", __FILE__)    
    File.expand_path("/tmp/#{file}", __FILE__)    
  end
  
  def create_importer file_name, suggested_name=nil, is_url=false<commit_msg>test working, but stops on each fail. need something smarter<commit_after># coding: UTF-8

require 'spec_helper'

describe CartoDB::Importer do
  homepath = File.expand_path('~')
  if File.exists? "#{homepath}/Dropbox/ec2-keys/.amazon_key"
    require 'aws-sdk'
    
    local_storage_dir = "/tmp/failed_files"
    unless File.directory? local_storage_dir
      Dir.mkdir(local_storage_dir)
    end
    
    s3 = AWS::S3.new({
      :access_key_id => File.open("#{homepath}/Dropbox/ec2-keys/.amazon_key", "rb").read.strip,
      :secret_access_key => File.open("#{homepath}/Dropbox/ec2-keys/.amazon_secret", "rb").read.strip
    })
    bucket = s3.buckets[:failed_imports]
    
    it "test failed files from S3" do
      #remote_object = bucket.objects['temp/wri-test-input_1334689661.csv']
      bucket.objects.each do |remote_object|
        if remote_object.content_length == 0
          remote_object.delete
          warn 'empty file, removing from tests'
        else
          key = remote_object.key
          name = key.split('/').last
          local_path = "#{local_storage_dir}/#{name}"
          p local_path
          #remote_object = bucket.objects[key]
          File.open(local_path,'w+').write(remote_object.read())
          importer = create_importer local_path, 'import_file'
          result   = importer.import!
          result.name.should          == 'import_file'
          if true
            File.delete(local_path)
            #remote_object.delete
          end
        end
      end
    end
  else 
    warn "You don't have Vizzuality S3 keys available."
    warn "If you aren't from Vizzuality, this is normal."
  end
  ##################################################
  # configuration & helpers for tests
  ##################################################
  before(:all) do
    @db = CartoDB::ImportDatabaseConnection.connection
    @db_opts = {:database => "cartodb_importer_test", 
                :username => "postgres", :password => '',
                :host => 'localhost', 
                :port => 5432}
  end
  
  after(:all) do
    CartoDB::ImportDatabaseConnection.drop
  end
  def create_importer file_name, suggested_name=nil, is_url=false
{'fraction_non_alphanumeric': 0.16279604506783169, 'fraction_numerical': 0.011037020004598759, 'mean_word_length': 2.670886075949367, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5372779', 'n_tokens_mistral': 1382, 'n_tokens_neox': 1322, 'n_words': 269}