<jupyter_start><jupyter_text>Load frequency info<jupyter_code>import json
def load_frequency_info(dataset: str="sst"):
    with open(ROOT / f"info/train_freqs_{dataset}.json", "rt") as f:
        freqs = json.load(f)
    return freqs
freqs = load_frequency_info(dataset)
words = list([x for x in freqs.keys() if x not in ["[CLS]", "[SEP]"]])<jupyter_output><empty_output><jupyter_text>Filter stop words that are too frequent<jupyter_code>words = [w for w in words if w in vocab.vocab]
import matplotlib.pyplot as plt
def plot_stats(xfunc, yfunc, figsize=(7, 7), **settings):
    fig, ax = plt.subplots(figsize=figsize)
    ax.set(**settings)
    ax.set_xlabel(xfunc.__name__)
    ax.set_ylabel(yfunc.__name__)
    ax.scatter(np.array([xfunc(w) for w in words]), np.array([yfunc(w) for w in words]))
    return fig, ax
import math
def freq(x):
    return freqs[x]
plot_stats(
    freq, norm,
    xscale="log",
)
def load_word_importance_info(dataset: str="sst"):
    with open(ROOT / f"info/word_positivities_{dataset}.json", "rt") as f:
        retval = json.load(f)
    return retval
word_importances = load_word_importance_info()
plot_stats(
    freq, lambda x: word_importances.get(x, 0),
    xscale='log'
)
plot_stats(
    lambda x: word_importances.get(x, 0), norm
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13758647194465795, 'fraction_numerical': 0.0030745580322828594, 'mean_word_length': 4.73568281938326, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '633456', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 497, 'n_tokens_neox': 488, 'n_words': 114, 'path': 'notebooks/analyze_model_embedding_relation_with_word_attributes.ipynb', 'repo_name': 'neulab/RIPPLe', 'size': '1301'}