<commit_before>$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))

require 'markov'

source_dir = ARGV[0]

markov = Markov.generator(3)

Dir["#{source_dir}/*.txt"].each do | f |
  puts "*** Analyzing '#{f}' "
  markov.parse_text f
end

#markov.dump_startwords
#markov.dump_dictionary

1..5.times do
  puts "\n#{markov.generate_sentence}"
end<commit_msg>added num of sentences to the bulk test<commit_after>$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))

require 'markov'

source_dir = ARGV[0]
num = ARGV[1].to_i

markov = Markov.generator(3)

Dir["#{source_dir}/*.txt"].each do | f |
  puts "*** Analyzing '#{f}' "
  markov.parse_text f
end

#markov.dump_startwords
#markov.dump_dictionary

1..num.times do
  puts "\n#{markov.generate_sentence}"
end
{'fraction_non_alphanumeric': 0.17834394904458598, 'fraction_numerical': 0.01019108280254777, 'mean_word_length': 5.046153846153846, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2454218', 'n_tokens_mistral': 318, 'n_tokens_neox': 312, 'n_words': 56}