<commit_before>require 'config'
require 'test/unit'
require 'mocha/test_unit'
require 'faker'
require 'veto'
require 'fileutils'
require 'json'

require 'toolshed'

Test::Unit.at_start do
  Toolshed::Client.use_git_submodules = false
  Toolshed::Client.git_quiet = '&> /dev/null'

  I18n.config.enforce_available_locales = true

  # setup a fake remote directory so we can reference everything locally
  system("rm -r -f #{File.join(TEST_ROOT, "remote")}")
  if (Dir.exists? (File.join(TEST_ROOT, "remote")))
    Dir.rmdir(File.join(TEST_ROOT, "remote"))
  end

  Dir.mkdir(File.join(TEST_ROOT, "remote"), 0777)
  Dir.chdir(File.join(TEST_ROOT, "remote"))

  # setup a couple of branches acting as the remote repository
  system('git init &> /dev/null')
  FileUtils.touch('file.txt')
  system('git add * &> /dev/null')
  system('git commit -m"Add empty file as commit" &> /dev/null')
  system('git checkout -b development master &> /dev/null')

  system("rm -r -f #{File.join(TEST_ROOT, "tmp")}")
  if (Dir.exists? (File.join(TEST_ROOT, "tmp")))
    Dir.rmdir(File.join(TEST_ROOT, "tmp"))
  end

  Dir.mkdir(File.join(TEST_ROOT, "tmp"), 0777)
  Dir.chdir(File.join(TEST_ROOT, "tmp"))


  # setup the new repository with an empty set this is configured in the config.rb file
  system("git init &> /dev/null")
  system("git remote add origin #{File.join(TEST_ROOT, "remote")} &> /dev/null")
  system('git fetch &> /dev/null')
  system('git checkout -t origin/master &> /dev/null')
end

Test::Unit.at_exit do
  FileUtils.rm_rf(File.join(TEST_ROOT, "remote"))
  FileUtils.rm_rf(File.join(TEST_ROOT, "tmp"))
end
<commit_msg>remove unneeded removes<commit_after>require 'config'
require 'test/unit'
require 'mocha/test_unit'
require 'faker'
require 'veto'
require 'fileutils'
require 'json'

require 'toolshed'

Test::Unit.at_start do
  Toolshed::Client.use_git_submodules = false
  Toolshed::Client.git_quiet = '&> /dev/null'

  I18n.config.enforce_available_locales = true

  # setup a fake remote directory so we can reference everything locally
  if (Dir.exists? (File.join(TEST_ROOT, "remote")))
    Dir.rmdir(File.join(TEST_ROOT, "remote"))
  end

  Dir.mkdir(File.join(TEST_ROOT, "remote"), 0777)
  Dir.chdir(File.join(TEST_ROOT, "remote"))

  # setup a couple of branches acting as the remote repository
  system('git init &> /dev/null')
  FileUtils.touch('file.txt')
  system('git add * &> /dev/null')
  system('git commit -m"Add empty file as commit" &> /dev/null')
  system('git checkout -b development master &> /dev/null')

  if (Dir.exists? (File.join(TEST_ROOT, "tmp")))
    Dir.rmdir(File.join(TEST_ROOT, "tmp"))
  end

  Dir.mkdir(File.join(TEST_ROOT, "tmp"), 0777)
  Dir.chdir(File.join(TEST_ROOT, "tmp"))


  # setup the new repository with an empty set this is configured in the config.rb file
  system("git init &> /dev/null")
  system("git remote add origin #{File.join(TEST_ROOT, "remote")} &> /dev/null")
  system('git fetch &> /dev/null')
  system('git checkout -t origin/master &> /dev/null')
end

Test::Unit.at_exit do
  FileUtils.rm_rf(File.join(TEST_ROOT, "remote"))
  FileUtils.rm_rf(File.join(TEST_ROOT, "tmp"))
end

{'fraction_non_alphanumeric': 0.15727475326329193, 'fraction_numerical': 0.006367398917542184, 'mean_word_length': 4.5808170515097695, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2704630', 'n_tokens_mistral': 1154, 'n_tokens_neox': 1111, 'n_words': 274}