<commit_before># encoding: utf-8
require 'fileutils'
require_relative '../../lib/importer/unp'
require_relative '../../../../spec/rspec_configuration.rb'

include CartoDB::Importer2

describe Unp do
  describe '#run' do
    it 'extracts the contents of the file' do
      zipfile   = zipfile_factory
      unp       = Unp.new

      unp.run(zipfile)
      Dir.entries(unp.temporary_directory).should include('bogus1.csv', 'bogus2.csv')
      FileUtils.rm_rf(unp.temporary_directory)
    end

    it 'extracts the contents of a carto file (see #11954)' do
      zipfile   = zipfile_factory(filename: 'this_is_a_zip_file.carto')
      unp       = Unp.new

      unp.run(zipfile)
      Dir.entries(unp.temporary_directory).should include('bogus1.csv', 'bogus2.csv')
      FileUtils.rm_rf(unp.temporary_directory)
    end

    it 'extracts the contents of a carto file with rar in the name (see #11954)' do
      zipfile   = zipfile_factory(filename: 'this_is_not_a_rar_file.carto')
      unp       = Unp.new

      unp.run(zipfile)
      Dir.entries(unp.temporary_directory).should include('bogus1.csv', 'bogus2.csv')
      FileUtils.rm_rf(unp.temporary_directory)
    end

    it 'populates a list of source files' do
      zipfile   = zipfile_factory
      unp       = Unp.new

      unp.source_files.should be_empty
      unp.run(zipfile)
      unp.source_files.should_not be_empty
    end

    it 'populates a single source file for the passed path if not compressed' do
      unp       = Unp.new

      unp.source_files.should be_empty
      unp.run(zipfile_factory)
      unp.source_files.length.should eq 2
    end
  end

  describe '#without_unpacking' do
    it 'pushes a source file for the passed file path to the source files' do
      unp       = Unp.new

      unp.source_files.should be_empty
      unp.without_unpacking(zipfile_factory)
      unp.source_files.size.should eq 1
    end

    it 'raises if the path does not belong to a file' do
      expect { Unp.new.without_unpacking('/var/tmp') }.to raise_error NotAFileError
    end
  end

  describe '#compressed?' do
    it 'returns true if extension denotes a compressed file' do
      unp       = Unp.new

      unp.compressed?('bogus.gz').should eq true
      unp.compressed?('bogus.csv').should eq false
    end
  end

  describe '#process' do
    it 'adds a source_file for the path if extension supported' do
      unp = Unp.new

      unp.source_files.should be_empty
      unp.process('/var/tmp/foo.csv')

      unp.source_files.should_not be_empty
      unp.source_files.first.should be_an_instance_of SourceFile
    end
  end

  describe '#crawl' do
    it 'returns a list of full paths for files in the directory' do
      fixture1  = '/var/tmp/bogus1.csv'
      fixture2  = '/var/tmp/bogus2.csv'
      FileUtils.touch(fixture1)
      FileUtils.touch(fixture2)

      unp       = Unp.new
      files     = unp.crawl('/var/tmp')

      files.should include(fixture1)
      files.should include(fixture2)

      FileUtils.rm(fixture1)
      FileUtils.rm(fixture2)
    end
  end

  describe '#extract' do
    it 'generates a temporary directory' do
      dir       = '/var/tmp/bogus'
      zipfile   = zipfile_factory(dir)
      unp       = Unp.new.extract(zipfile)

      File.directory?(unp.temporary_directory).should eq true

      FileUtils.rm_r(dir)
    end

    it 'extracts the contents of the file into the temporary directory' do
      dir       = '/var/tmp/bogus'
      zipfile   = zipfile_factory(dir)
      unp       = Unp.new.extract(zipfile)<commit_msg>Fix unp test scanning /var/tmp<commit_after># encoding: utf-8
require 'fileutils'
require_relative '../../lib/importer/unp'
require_relative '../../../../spec/rspec_configuration.rb'

include CartoDB::Importer2

describe Unp do
  describe '#run' do
    it 'extracts the contents of the file' do
      zipfile   = zipfile_factory
      unp       = Unp.new

      unp.run(zipfile)
      Dir.entries(unp.temporary_directory).should include('bogus1.csv', 'bogus2.csv')
      FileUtils.rm_rf(unp.temporary_directory)
    end

    it 'extracts the contents of a carto file (see #11954)' do
      zipfile   = zipfile_factory(filename: 'this_is_a_zip_file.carto')
      unp       = Unp.new

      unp.run(zipfile)
      Dir.entries(unp.temporary_directory).should include('bogus1.csv', 'bogus2.csv')
      FileUtils.rm_rf(unp.temporary_directory)
    end

    it 'extracts the contents of a carto file with rar in the name (see #11954)' do
      zipfile   = zipfile_factory(filename: 'this_is_not_a_rar_file.carto')
      unp       = Unp.new

      unp.run(zipfile)
      Dir.entries(unp.temporary_directory).should include('bogus1.csv', 'bogus2.csv')
      FileUtils.rm_rf(unp.temporary_directory)
    end

    it 'populates a list of source files' do
      zipfile   = zipfile_factory
      unp       = Unp.new

      unp.source_files.should be_empty
      unp.run(zipfile)
      unp.source_files.should_not be_empty
    end

    it 'populates a single source file for the passed path if not compressed' do
      unp       = Unp.new

      unp.source_files.should be_empty
      unp.run(zipfile_factory)
      unp.source_files.length.should eq 2
    end
  end

  describe '#without_unpacking' do
    it 'pushes a source file for the passed file path to the source files' do
      unp       = Unp.new

      unp.source_files.should be_empty
      unp.without_unpacking(zipfile_factory)
      unp.source_files.size.should eq 1
    end

    it 'raises if the path does not belong to a file' do
      expect { Unp.new.without_unpacking('/var/tmp') }.to raise_error NotAFileError
    end
  end

  describe '#compressed?' do
    it 'returns true if extension denotes a compressed file' do
      unp       = Unp.new

      unp.compressed?('bogus.gz').should eq true
      unp.compressed?('bogus.csv').should eq false
    end
  end

  describe '#process' do
    it 'adds a source_file for the path if extension supported' do
      unp = Unp.new

      unp.source_files.should be_empty
      unp.process('/var/tmp/foo.csv')

      unp.source_files.should_not be_empty
      unp.source_files.first.should be_an_instance_of SourceFile
    end
  end

  describe '#crawl' do
    it 'returns a list of full paths for files in the directory' do
      Dir.mktmpdir do |path|
        fixture1  = "#{path}/bogus1.csv"
        fixture2  = "#{path}/bogus2.csv"
        FileUtils.touch(fixture1)
        FileUtils.touch(fixture2)

        unp       = Unp.new
        files     = unp.crawl(path)

        files.should include(fixture1)
        files.should include(fixture2)
      end
    end
  end

  describe '#extract' do
    it 'generates a temporary directory' do
      dir       = '/var/tmp/bogus'
      zipfile   = zipfile_factory(dir)
      unp       = Unp.new.extract(zipfile)

      File.directory?(unp.temporary_directory).should eq true

      FileUtils.rm_r(dir)
    end

    it 'extracts the contents of the file into the temporary directory' do
      dir       = '/var/tmp/bogus'
      zipfile   = zipfile_factory(dir)
      unp       = Unp.new.extract(zipfile)
{'fraction_non_alphanumeric': 0.09124241996897475, 'fraction_numerical': 0.008179382315611338, 'mean_word_length': 2.498766650222003, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1773896', 'n_tokens_mistral': 2448, 'n_tokens_neox': 2380, 'n_words': 591}