<commit_before>import optparse
import os
from collections import defaultdict

optparser = optparse.OptionParser()
optparser.add_option("-d", "--directory", dest="directory", default="uighur/", help="Directory with dictionaries")
(opts, _) = optparser.parse_args()

full_path = os.path.abspath(opts.directory)
all_files = set([filename for filename in os.listdir(full_path)])

all_uighur_words = set([])

for line in open(full_path + '/' + 'all_lexicons', encoding='utf-8'):
    lexicons = line.strip().split('\t')
    all_uighur_words.add(lexicons[0])

webcrawl_content = open(full_path + '/' + 'webcrawl_no-en_no-zh.ug', encoding='utf-8').read()

uighur_token_frequency = defaultdict(int)

for token in webcrawl_content.split(' '):
    if token in all_uighur_words:
        uighur_token_frequency[token] += 1

with open(full_path + '/dict.ug', 'w', encoding='utf-8') as text_file:
    text_file.write("\n".join(uighur_token_frequency.keys()))<commit_msg>Update script for creating uighur dictionary to pull in the translations, and process them a little<commit_after>import optparse
import os
from collections import defaultdict

optparser = optparse.OptionParser()
optparser.add_option("-d", "--directory", dest="directory", default="uighur/", help="Directory with dictionaries")
(opts, _) = optparser.parse_args()

full_path = os.path.abspath(opts.directory)
all_files = set([filename for filename in os.listdir(full_path)])

all_uighur_words = defaultdict(list)

for line in open(full_path + '/' + 'all_lexicons', encoding='utf-8'):
    lexicons = line.strip().split('\t')
    if len(lexicons) == 3:
        # lets process this a little so everything is not trans1, trans2, trans3.
        translations_string = lexicons[2]
        if translations_string.endswith("."):
            translations_string = translations_string[:-1]
        translations_array = [t.strip() for t in translations_string.split(',')]
        translations_array = [t for t in translations_array if len(t) > 0]
        all_uighur_words[lexicons[0]] += translations_array
    else:
        all_uighur_words[lexicons[0]] = []

webcrawl_content = open(full_path + '/' + 'webcrawl_no-en_no-zh.ug', encoding='utf-8').read()

uighur_token_frequency = defaultdict(int)

for token in webcrawl_content.split(' '):
    if token in all_uighur_words.keys():
        uighur_token_frequency[token] += 1

tab_separated_uighur_and_translation = [ "\t".join([k]+all_uighur_words[k]) for k in uighur_token_frequency.keys()]

with open(full_path + '/dict.ug', 'w', encoding='utf-8') as text_file:
    text_file.write("\n".join(tab_separated_uighur_and_translation))
{'fraction_non_alphanumeric': 0.13986280487804878, 'fraction_numerical': 0.006859756097560976, 'mean_word_length': 4.993150684931507, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6035540', 'n_tokens_mistral': 896, 'n_tokens_neox': 876, 'n_words': 183}