<commit_before>require "language_pack"
require "pathname"
require "yaml"
require "digest/sha1"
require "language_pack/shell_helpers"
require "language_pack/cache"

Encoding.default_external = Encoding::UTF_8 if defined?(Encoding)

# abstract class that all the Ruby based Language Packs inherit from
class LanguagePack::Base
  include LanguagePack::ShellHelpers

  VENDOR_URL = "https://s3.amazonaws.com/heroku-buildpack-ruby"

  attr_reader :build_path, :cache

  # changes directory to the build_path
  # @param [String] the path of the build dir
  # @param [String] the path of the cache dir
  def initialize(build_path, cache_path=nil)
    @build_path = build_path
    @cache      = LanguagePack::Cache.new(cache_path) if cache_path
    @id         = Digest::SHA1.hexdigest("#{Time.now.to_f}-#{rand(1000000)}")[0..10]

    Dir.chdir build_path
  end

  def self.===(build_path)
    raise "must subclass"
  end

  # name of the Language Pack
  # @return [String] the result
  def name
    raise "must subclass"
  end

  # list of default addons to install
  def default_addons
    raise "must subclass"
  end
<commit_msg>document when cache_dir is nil<commit_after>require "language_pack"
require "pathname"
require "yaml"
require "digest/sha1"
require "language_pack/shell_helpers"
require "language_pack/cache"

Encoding.default_external = Encoding::UTF_8 if defined?(Encoding)

# abstract class that all the Ruby based Language Packs inherit from
class LanguagePack::Base
  include LanguagePack::ShellHelpers

  VENDOR_URL = "https://s3.amazonaws.com/heroku-buildpack-ruby"

  attr_reader :build_path, :cache

  # changes directory to the build_path
  # @param [String] the path of the build dir
  # @param [String] the path of the cache dir this is nil during detect
  def initialize(build_path, cache_path=nil)
    @build_path = build_path
    @cache      = LanguagePack::Cache.new(cache_path) if cache_path
    @id         = Digest::SHA1.hexdigest("#{Time.now.to_f}-#{rand(1000000)}")[0..10]

    Dir.chdir build_path
  end

  def self.===(build_path)
    raise "must subclass"
  end

  # name of the Language Pack
  # @return [String] the result
  def name
    raise "must subclass"
  end

  # list of default addons to install
  def default_addons
    raise "must subclass"
  end

{'fraction_non_alphanumeric': 0.09515495416848538, 'fraction_numerical': 0.012221737232649499, 'mean_word_length': 4.093333333333334, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3483336', 'n_tokens_mistral': 752, 'n_tokens_neox': 725, 'n_words': 217}