<commit_before>require "unindent"
require 'open3'

describe "CLI" do
  before :all do
    # use local scripts
    ENV["PATH"] = "#{File.join(File.dirname(__FILE__),"..","bin")}:#{ENV["PATH"]}"
  end

  def run(cmd, options={})
    Open3.popen2e(cmd) do |stdin, stdout_and_stderr, wait_thr|
      unless wait_thr.value.success?
        message = "Unable to run #{cmd.inspect} in #{Dir.pwd}.\n#{stdout_and_stderr.read}"
        warn "ERROR: #{message}"
        raise message unless options[:fail]
      end
      stdout_and_stderr.read
    end
  end

  def write(file, content)
    File.open(file, 'w'){|f| f.write content }
  end

  around do |example|
    dir = "spec/tmp"
    run "rm -rf #{dir}"
    run "mkdir #{dir}"

    # use fake home for .ssh hacks
    run "mkdir #{dir}/home"
    ENV["HOME"] = File.expand_path("#{dir}/home")
<commit_msg>Fix a bug in the run helper<commit_after>require "unindent"
require 'open3'

describe "CLI" do
  before :all do
    # use local scripts
    ENV["PATH"] = "#{File.join(File.dirname(__FILE__),"..","bin")}:#{ENV["PATH"]}"
  end

  def run(cmd, options={})
    Open3.popen2e(cmd) do |stdin, stdout_and_stderr, wait_thr|
      output = stdout_and_stderr.read
      return output if wait_thr.value.success?
      return output if options[:fail]

      message = "Unable to run #{cmd.inspect} in #{Dir.pwd}.\n#{output}"
      warn "ERROR: #{message}"
      raise message unless options[:fail]
    end
  end

  def write(file, content)
    File.open(file, 'w'){|f| f.write content }
  end

  around do |example|
    dir = "spec/tmp"
    run "rm -rf #{dir}"
    run "mkdir #{dir}"

    # use fake home for .ssh hacks
    run "mkdir #{dir}/home"
    ENV["HOME"] = File.expand_path("#{dir}/home")

{'fraction_non_alphanumeric': 0.16224018475750576, 'fraction_numerical': 0.003464203233256351, 'mean_word_length': 3.0872641509433962, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4505161', 'n_tokens_mistral': 632, 'n_tokens_neox': 609, 'n_words': 162}