<commit_before>History.txt
LICENSE.txt
Manifest.txt
README.txt
Rakefile
lib/raingrams.rb
lib/raingrams/version.rb
lib/raingrams/raingrams.rb
lib/raingrams/exceptions/prefixfrequencymissing.rb
lib/raingrams/exceptions.rb
lib/raingrams/extensions/class.rb
lib/raingrams/extensions/falseclass.rb
lib/raingrams/extensions/nilclass.rb
lib/raingrams/extensions/object.rb
lib/raingrams/extensions/string.rb
lib/raingrams/extensions/symbol.rb
lib/raingrams/extensions/trueclass.rb
lib/raingrams/extensions.rb
lib/raingrams/tokens/token.rb
lib/raingrams/tokens/startsentence.rb
lib/raingrams/tokens/stopsentence.rb
lib/raingrams/tokens/unknown.rb
lib/raingrams/tokens.rb
lib/raingrams/ngram.rb
lib/raingrams/model.rb
lib/raingrams/unigrammodel.rb
lib/raingrams/multigrammodel.rb
lib/raingrams/bigrammodel.rb
lib/raingrams/trigrammodel.rb
lib/raingrams/quadgrammodel.rb
lib/raingrams/pentagrammodel.rb
lib/raingrams/hexagrammodel.rb
lib/raingrams/openvocabulary/openmodel.rb
lib/raingrams/openvocabulary/unigrammodel.rb
lib/raingrams/openvocabulary/multigrammodel.rb
lib/raingrams/openvocabulary/bigrammodel.rb
lib/raingrams/openvocabulary/trigrammodel.rb
lib/raingrams/openvocabulary/quadgrammodel.rb
lib/raingrams/openvocabulary/pentagrammodel.rb
lib/raingrams/openvocabulary/hexagrammodel.rb
lib/raingrams/openvocabulary.rb
test/test_raingrams.rb<commit_msg>* Updated Manifest for underscored file names.<commit_after>History.txt
LICENSE.txt
Manifest.txt
README.txt
Rakefile
lib/raingrams.rb
lib/raingrams/version.rb
lib/raingrams/raingrams.rb
lib/raingrams/exceptions/prefix_frequency_missing.rb
lib/raingrams/exceptions.rb
lib/raingrams/extensions/class.rb
lib/raingrams/extensions/false_class.rb
lib/raingrams/extensions/nil_class.rb
lib/raingrams/extensions/object.rb
lib/raingrams/extensions/string.rb
lib/raingrams/extensions/symbol.rb
lib/raingrams/extensions/true_class.rb
lib/raingrams/extensions.rb
lib/raingrams/tokens/token.rb
lib/raingrams/tokens/start_sentence.rb
lib/raingrams/tokens/stop_sentence.rb
lib/raingrams/tokens/unknown.rb
lib/raingrams/tokens.rb
lib/raingrams/ngram.rb
lib/raingrams/model.rb
lib/raingrams/unigram_model.rb
lib/raingrams/multigram_model.rb
lib/raingrams/bigram_model.rb
lib/raingrams/trigram_model.rb
lib/raingrams/quadgram_model.rb
lib/raingrams/pentagram_model.rb
lib/raingrams/hexagram_model.rb
lib/raingrams/open_vocabulary/open_model.rb
lib/raingrams/open_vocabulary/unigram_model.rb
lib/raingrams/open_vocabulary/multigram_model.rb
lib/raingrams/open_vocabulary/bigram_model.rb
lib/raingrams/open_vocabulary/trigram_model.rb
lib/raingrams/open_vocabulary/quadgram_model.rb
lib/raingrams/open_vocabulary/pentagram_model.rb
lib/raingrams/open_vocabulary/hexagram_model.rb
lib/raingrams/open_vocabulary.rb
test/test_raingrams.rb
{'fraction_non_alphanumeric': 0.09906001446131597, 'fraction_numerical': 0.0, 'mean_word_length': 6.772471910112359, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7407278', 'n_tokens_mistral': 1146, 'n_tokens_neox': 1060, 'n_words': 89}