<jupyter_start><jupyter_text>Prepare the dictionary (using CSW + freq. data from Kaggle dataset)<jupyter_code>import os
import pandas as pd

base_path = ".."
data_path = os.path.join(base_path, "data")<jupyter_output><empty_output><jupyter_text>Use the [Collins Scrabble Words](https://en.wikipedia.org/wiki/Collins_Scrabble_Words) (CSW) 2019 dictionary, but sort by the frequency estimates from the [Google Web Trillion Word Corpus](https://www.kaggle.com/rtatman/english-word-frequency).<jupyter_code>kaggle_words = pd.read_csv(os.path.join(data_path, "unigram_freq.csv"))
kaggle_words["freq"] = kaggle_words["count"] / kaggle_words["count"].sum()

csw_words = pd.read_csv(
    os.path.join(data_path, "csw_2019.txt"), skiprows=2, header=None, names=["word"]
)
csw_words.word = csw_words.word.str.lower()
csw_with_freq = (
    csw_words.join(kaggle_words.set_index("word"), on="word", how="left")
    .sort_values("freq", ascending=False)
    .reset_index(drop=True)
)
len(csw_with_freq)
csw_with_freq
csw_with_freq.to_csv(os.path.join(data_path, "dictionary.csv"), index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14954954954954955, 'fraction_numerical': 0.008108108108108109, 'mean_word_length': 7.050724637681159, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '547315', 'license': 'MIT', 'n_tokens_mistral': 442, 'n_tokens_neox': 424, 'n_words': 65, 'path': 'notebook/prepare_dictionary.ipynb', 'repo_name': 'ttesileanu/clonle', 'size': '1110'}