<commit_before>$:.unshift(File.dirname(__FILE__)) unless
$:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))

module Stead
  VERSION = '0.0.1'

  CONTAINER_TYPES = [
    "album",
    "artifactbox",
    "audiocassette",
    "audiotape",
    "box",
    "cardbox",
    "carton",
    "cassette",
    "cassettebox",
    "cdbox",
    "diskette",
    "drawingsbox",
    "flatbox",
    "flatfile",
    "flatfolder",
    "folder",
    "halfbox",
    "item",
    "largeenvelope",
    "legalbox",
    "mapcase",
    "mapfolder",
    "notecardbox",
    "othertype",
    "oversize",
    "oversizebox",
    "oversizeflatbox",
    "reel",
    "reelbox",
    "scrapbook",
    "slidebox",
    "tube",
    "tubebox",
    "videotape",
    "volume"
  ]
end

require 'rubygems'
require 'nokogiri'
require 'csv'
if CSV.const_defined? :Reader<commit_msg>add drawer as a valid container type<commit_after>$:.unshift(File.dirname(__FILE__)) unless
$:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))

module Stead
  VERSION = '0.0.1'

  CONTAINER_TYPES = [
    "album",
    "artifactbox",
    "audiocassette",
    "audiotape",
    "box",
    "cardbox",
    "carton",
    "cassette",
    "cassettebox",
    "cdbox",
    "diskette",
    "drawer",
    "drawingsbox",
    "flatbox",
    "flatfile",
    "flatfolder",
    "folder",
    "halfbox",
    "item",
    "largeenvelope",
    "legalbox",
    "mapcase",
    "mapfolder",
    "notecardbox",
    "othertype",
    "oversize",
    "oversizebox",
    "oversizeflatbox",
    "reel",
    "reelbox",
    "scrapbook",
    "slidebox",
    "tube",
    "tubebox",
    "videotape",
    "volume"
  ]
end

require 'rubygems'
require 'nokogiri'
require 'csv'
if CSV.const_defined? :Reader
{'fraction_non_alphanumeric': 0.17574396406513196, 'fraction_numerical': 0.003368893879842785, 'mean_word_length': 2.995515695067265, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2166188', 'n_tokens_mistral': 707, 'n_tokens_neox': 690, 'n_words': 114}