<commit_before>import re
import random

words = ''

with open('seed.txt', 'r') as seed:
	for line in seed:
		words += line

wordlist = re.findall(r"[\w']+", words)

worddict = dict()

prev = ''
for word in wordlist:
	if prev is '':
		prev = word
		continue

	if prev in worddict:
		worddict[prev].append(word)
	else:
		worddict[prev] = [word]

	prev = word

word = random.choice(list(worddict.keys()))

sentence = word
while True:
	if word not in list(worddict.keys()): break
	word = random.choice(worddict[word])
	sentence += ' ' + word

print(sentence)<commit_msg>Getting some pretty good output now<commit_after>import re
import random

words = ''

with open('seed.txt', 'r') as seed:
	for line in seed:
		words += line

wordlist = re.findall(r"[\w']+[\".,;:]?", words)

worddict = dict()

prev = (wordlist[0], wordlist[1])
for i in range(2, len(wordlist)):
	word = wordlist[i]
	if prev in worddict:
		worddict[prev].append(word)
	else:
		worddict[prev] = [word]

	prev = (prev[1], word)

prefix = 'a'
while not prefix[0].isupper():
	prefix = random.choice(list(worddict.keys()))

i = 0
sentence = prefix[0] + ' ' + prefix[1]
while True:
	if prefix not in list(worddict.keys()):
		sentence += '.'
		break
	if '.' in prefix[1]: break
	prefix = (prefix[1], random.choice(worddict[prefix]))
	sentence += ' ' + prefix[1]
	i += 1

print(sentence)
{'fraction_non_alphanumeric': 0.17038690476190477, 'fraction_numerical': 0.008928571428571428, 'mean_word_length': 4.153256704980843, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2683468', 'n_tokens_mistral': 522, 'n_tokens_neox': 497, 'n_words': 138}