<commit_before>module Thinner

  # The base location of the Thinner gem.
  ROOT               = File.expand_path "#{File.dirname __FILE__}/.."

  # The Thinner version.
  VERSION            = File.read("#{ROOT}/VERSION").chomp

  # The process label to run each Thinner::Client under.
  PROCESS_IDENTIFIER = "Thinner"

  # Set up the configuration instance as a class level accessor.
  class << self; attr_accessor :configuration; end

  # Set any thinner settings by passing in a block.
  def self.configure
    self.configuration ||= Configuration.new
    yield configuration
  end

  # Halt any running instances of Thinner::Client
  def self.stop!
    Purger.stop!
  end

  # Begin purging urls.
  def self.purge! urls
    Purger.new(urls).purge!
  end

end

# Various requires and dependencies
require "klarlack"
require "logger"
require "#{Thinner::ROOT}/lib/thinner/configuration"
require "#{Thinner::ROOT}/lib/thinner/client"
require "#{Thinner::ROOT}/lib/thinner/purger"<commit_msg>removed a no duh line<commit_after>module Thinner

  # The base location of the Thinner gem.
  ROOT               = File.expand_path "#{File.dirname __FILE__}/.."

  # The Thinner version.
  VERSION            = File.read("#{ROOT}/VERSION").chomp

  # The process label to run each Thinner::Client under.
  PROCESS_IDENTIFIER = "Thinner"

  # Set up the configuration instance as a class level accessor.
  class << self; attr_accessor :configuration; end

  # Set any thinner settings by passing in a block.
  def self.configure
    self.configuration ||= Configuration.new
    yield configuration
  end

  # Halt any running instances of Thinner::Client
  def self.stop!
    Purger.stop!
  end

  # Begin purging urls.
  def self.purge! urls
    Purger.new(urls).purge!
  end

end

require "klarlack"
require "logger"
require "#{Thinner::ROOT}/lib/thinner/configuration"
require "#{Thinner::ROOT}/lib/thinner/client"
require "#{Thinner::ROOT}/lib/thinner/purger"
{'fraction_non_alphanumeric': 0.10388945752302968, 'fraction_numerical': 0.0, 'mean_word_length': 3.463470319634703, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6909005', 'n_tokens_mistral': 615, 'n_tokens_neox': 592, 'n_words': 183}