<commit_before>module RCR
	module LanguageModel
		class Base
			# Return a score of next letter given a word.
			def score(context, continuation)
				raise "Not implemented."
			end

			# Return the most likely word given a list of letter classification
			# results (array of hashes of letter => score).
			# Default implementation: Viterbi algorithm.
			def most_likely_words(letter_hypotheses, want_words: 10)
				last_most_likely_words = { 1.0 => "" }

				# TODO: probably doesn't actually work all that well... !!!!!
				# TODO: ALE TO BY ZNAMENALO ZE <NAME> VITERBIHO I JINDE!

				letter_hypotheses.each do |letter_hypothesis|
					# letter hypothesis: hash of letter => score
					best_now = {}

					# pp last_most_likely_words
					# pp letter_hypothesis

					letter_hypothesis.each do |letter, letter_score|
						last_most_likely_words.each do |prefix_probability, prefix|
							# puts "Prefix: #{prefix.inspect}"
							posterior_score = letter_score * score(prefix, letter)
							my_score = posterior_score * prefix_probability

							best_now[my_score] = prefix + letter
							# if best.nil? || best_score < my_score
							# 	best, best_score = letter, my_score
							# end
						end
					end

					good_enough = best_now.keys.sort.reverse.take(want_words)
					keep = good_enough.last
					last_most_likely_words = best_now.delete_if { |key, value| key < keep }

					pp last_most_likely_words
				end<commit_msg>vetsi prostor na slova<commit_after>module RCR
	module LanguageModel
		class Base
			# Return a score of next letter given a word.
			def score(context, continuation)
				raise "Not implemented."
			end

			# Return the most likely word given a list of letter classification
			# results (array of hashes of letter => score).
			# Default implementation: Viterbi algorithm.
			def most_likely_words(letter_hypotheses, want_words: 100)
				last_most_likely_words = { 1.0 => "" }

				# TODO: probably doesn't actually work all that well... !!!!!
				# TODO: ALE TO BY <NAME> I JINDE!

				letter_hypotheses.each do |letter_hypothesis|
					# letter hypothesis: hash of letter => score
					best_now = {}

					# pp last_most_likely_words
					# pp letter_hypothesis

					letter_hypothesis.each do |letter, letter_score|
						last_most_likely_words.each do |prefix_probability, prefix|
							# puts "Prefix: #{prefix.inspect}"
							posterior_score = letter_score * score(prefix, letter)
							my_score = posterior_score * prefix_probability

							best_now[my_score] = prefix + letter
							# if best.nil? || best_score < my_score
							# 	best, best_score = letter, my_score
							# end
						end
					end

					good_enough = best_now.keys.sort.reverse.take(want_words)
					keep = good_enough.last
					last_most_likely_words = best_now.delete_if { |key, value| key < keep }

					pp last_most_likely_words
				end
{'fraction_non_alphanumeric': 0.08109052778748689, 'fraction_numerical': 0.0031457532331352674, 'mean_word_length': 3.071123755334282, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7462210', 'n_tokens_mistral': 1163, 'n_tokens_neox': 869, 'n_words': 253}