<commit_before># encoding: utf-8

require 'rake'
require 'spec_helper'

describe 'Packaging and running a project' do
  def isolated_run(dir, cmd)
    Dir.chdir(dir) do
      Bundler.clean_system("PATH=$GEM_HOME/bin:$PWD/../../../tmp/hadoop-current/bin:$PATH HUMBOLDT_TEST_PROJECT_PATH=$PWD rvm $RUBY_VERSION@humboldt-test_project do #{cmd}")
    end
  end

  let :test_project_dir do
    File.expand_path('../../fixtures/test_project', __FILE__)
  end

  before :all do
    FileUtils.rm_rf File.join(test_project_dir, 'build')
    FileUtils.rm_rf File.join(test_project_dir, 'data/test_project/output')
    FileUtils.rm_rf File.join(test_project_dir, 'data/log')
    FileUtils.rm_rf File.join(test_project_dir, 'important.doc')
    FileUtils.rm_rf File.join(test_project_dir, 'another_file')
    FileUtils.rm_rf File.join(test_project_dir, 'Gemfile.lock')
  end

  around do |example|
    Dir.chdir(test_project_dir) do
      example.run
    end
  end

  context 'Running the project' do
    before :all do
      isolated_run(test_project_dir, "gem install ../../../pkg/*.gem")
      isolated_run(test_project_dir, "bundle install")
      isolated_run(test_project_dir, "bundle exec humboldt run-local --cleanup-before --data-path=data --input='input/*' 2>&1 | tee data/log")
    end

    context 'file caching' do
      it 'successfully reads cached files from mapper' do
        File.readlines('data/test_project/output/part-r-00000').first.should == "mapper\tsecrit/not so secrit\n"
      end

      it 'successfully reads cached files from reducer' do
        File.readlines('data/test_project/output/part-r-00000').last.should == "reducer\tsecrit/not so secrit\n"
      end
    end
  end
end
<commit_msg>Specify that humboldt jar is packaged.<commit_after># encoding: utf-8

require 'rake'
require 'spec_helper'

describe 'Packaging and running a project' do
  def isolated_run(dir, cmd)
    Dir.chdir(dir) do
      Bundler.clean_system("PATH=$GEM_HOME/bin:$PWD/../../../tmp/hadoop-current/bin:$PATH HUMBOLDT_TEST_PROJECT_PATH=$PWD rvm $RUBY_VERSION@humboldt-test_project do #{cmd}")
    end
  end

  let :test_project_dir do
    File.expand_path('../../fixtures/test_project', __FILE__)
  end

  before :all do
    FileUtils.rm_rf File.join(test_project_dir, 'build')
    FileUtils.rm_rf File.join(test_project_dir, 'data/test_project/output')
    FileUtils.rm_rf File.join(test_project_dir, 'data/log')
    FileUtils.rm_rf File.join(test_project_dir, 'important.doc')
    FileUtils.rm_rf File.join(test_project_dir, 'another_file')
    FileUtils.rm_rf File.join(test_project_dir, 'Gemfile.lock')

    isolated_run(test_project_dir, "gem install ../../../pkg/*.gem")
    isolated_run(test_project_dir, "bundle install")
    isolated_run(test_project_dir, "bundle exec humboldt package")
  end

  around do |example|
    Dir.chdir(test_project_dir) do
      example.run
    end
  end

  context 'Project package' do
    let :jar do
      Java::JavaUtilJar::JarFile.new(Java::JavaIo::File.new(File.expand_path('build/test_project.jar')))
    end

    let :jar_entries do
      jar.entries.to_a.map(&:name)
    end

    it 'includes the built humboldt JAR file' do
      jar_entries.should include('lib/humboldt.jar')
    end
  end

  context 'Running the project' do
    before :all do
      isolated_run(test_project_dir, "bundle exec humboldt run-local --cleanup-before --skip-package --data-path=data --input='input/*' 2>&1 | tee data/log")
    end

    context 'file caching' do
      it 'successfully reads cached files from mapper' do
        File.readlines('data/test_project/output/part-r-00000').first.should == "mapper\tsecrit/not so secrit\n"
      end

      it 'successfully reads cached files from reducer' do
        File.readlines('data/test_project/output/part-r-00000').last.should == "reducer\tsecrit/not so secrit\n"
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.12248062015503876, 'fraction_numerical': 0.006718346253229974, 'mean_word_length': 3.5594817432273262, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4315659', 'n_tokens_mistral': 1367, 'n_tokens_neox': 1326, 'n_words': 283}