<commit_before># TODO

# 1. wort aus de-en rausextrahieren

# für D- wort: anfang der zeile, wort mit mind 5 buchstaben
# für EN-wort: wort ab ":: ", auch mit mind. 5 buchstaben
# kommentierte zeilen ignorieren

def analysewords(wordlist):
	alphabet = "abcdefghijklmnopqrstuvwxyz"
	goodwords = [] # empty list

	myfile = open("de-en.dict")
	for line in myfile:
		myword = line.split()[0]
		for letter in myword.lower():
			if len(myword) < 5:
				break
			if not letter in alphabet:
				break
		else:
			goodwords.append(myword)

	myfile.close()

	# occurence of letters in all words
	d = {}
	for word in goodwords:
		for letter in word.lower():
			if letter in d:
				d[letter] += 1
			else:
				d[letter] = 1

	# list of pairs (in the dictionary) sorted by occurence descending
	pairlist = sorted(d.items(), key=lambda x: x[1], reverse=True)

	# pairlist looks like this: [('e', 167410), ('n', 100164),...]
	occurencestring = ''

	for pair in pairlist:
		occurencestring += pair[0]

	tiplist = list(occurencestring)
<commit_msg>cleaned up dict-analyser file<commit_after>alphabet = "abcdefghijklmnopqrstuvwxyz" # allowed characters
goodwords = [] # empty list

myfile = open("de-en.dict")

for line in myfile:
	myword = line.split()[0]
	for letter in myword.lower():
		if len(myword) < 5: # only use words that are at least 5 characters long
			break
		if not letter in alphabet:
			break
	else:
		goodwords.append(myword)

myfile.close()

# create dictionary with occurence of letters in all words
mydict = {}
for word in goodwords:
	for letter in word.lower():
		if letter in mydict:
			mydict[letter] += 1
		else:
			mydict[letter] = 1

# list of pairs (in the dictionary) sorted by occurence (descending)
# adapted from http://stackoverflow.com/questions/613183/python-sort-a-dictionary-by-value
pairlist = sorted(mydict.items(), key=lambda x: x[1], reverse=True)
# pairlist looks like this: [('e', 167410), ('n', 100164),...]
occurencestring = ''

for pair in pairlist:
	occurencestring += pair[0]

tiplist = list(occurencestring)
print(tiplist)

{'fraction_non_alphanumeric': 0.12128592303945446, 'fraction_numerical': 0.022406234778373113, 'mean_word_length': 3.9613526570048307, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5852985', 'n_tokens_mistral': 819, 'n_tokens_neox': 732, 'n_words': 228}