<commit_before>$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)

if ENV["CODECLIMATE_REPO_TOKEN"]
  # report coverage only for latest mri ruby
  if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.2.0"
    require "codeclimate-test-reporter"
    CodeClimate::TestReporter.start
  end
else
  require "simplecov"
  SimpleCov.start
end

require "i18n/backend/advanced"
require "yaml"
begin
  require "pry"
rescue LoadError
end

RSpec.configure do |config|
  # begin --- rspec 3.1 generator
  config.expect_with :rspec do |expectations|
    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
  end

  config.mock_with :rspec do |mocks|
    mocks.verify_partial_doubles = true
  end
  # end --- rspec 3.1 generator
end

def assets_path
  File.expand_path(File.join(File.dirname(__FILE__), "assets"))
end

def read_asset(path_to_file)
  File.read(assets_path, path_to_file)
end

<commit_msg>Removed unused code from spec_helper<commit_after>$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)

if ENV["CODECLIMATE_REPO_TOKEN"]
  # report coverage only for latest mri ruby
  if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.2.0"
    require "codeclimate-test-reporter"
    CodeClimate::TestReporter.start
  end
else
  require "simplecov"
  SimpleCov.start
end

require "i18n/backend/advanced"
begin
  require "pry"
rescue LoadError
end

RSpec.configure do |config|
  # begin --- rspec 3.1 generator
  config.expect_with :rspec do |expectations|
    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
  end

  config.mock_with :rspec do |mocks|
    mocks.verify_partial_doubles = true
  end
  # end --- rspec 3.1 generator
end

def assets_path
  File.expand_path(File.join(File.dirname(__FILE__), "assets"))
end

{'fraction_non_alphanumeric': 0.10352673492605233, 'fraction_numerical': 0.010238907849829351, 'mean_word_length': 4.412307692307692, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4697080', 'n_tokens_mistral': 643, 'n_tokens_neox': 622, 'n_words': 141}