<jupyter_start><jupyter_text>Overview of Gold LexicaThis notebooks loads all datasets used in the gold evaluation, looks up their sizes and saves a csv table with the information to the disk. At the the end, the notebook produces a Latex-formatted version of the the Table.<jupyter_code>import numpy as np
import pandas as pd

from memolon.src import utils,constants
df = pd.DataFrame(columns=['id', 'variables', 'size', 'citation'])
for key, value in utils.conditions.items():
    try:
        data = value['get']()
        size = len(data)
        variables = "".join([c[0] for c in data.columns])
        citation = value['citation']
    except KeyError:
        print('Problem processing {}'.format(key))
    df.loc[len(df)] = [key, variables, size, citation]
df.to_csv(constants.GOLD_LEXICA_OVERVIEW, index=False)
df = pd.read_csv(constants.GOLD_LEXICA_OVERVIEW, index_col=0)
df.head()
print(df.to_latex())<jupyter_output>\begin{tabular}{llrl}
\toprule
{} & variables &   size &         citation \\
id  &           &        &                  \\
\midrule
en1 &       vad &   1032 &            xanew \\
en2 &       vad &   1034 &        bradley99 \\
en3 &     jasfd &   1034 &      stevenson07 \\
es1 &       vad &   1034 &        redondo07 \\
es2 &        va &  14031 &     Stadthagen16 \\
es3 &        va &    875 &        hiojosa16 \\
es4 &     jasfd &    875 &      hinojosa 16 \\
es5 &     jasfd &  10491 &     Stadthagen17 \\
es6 &     jasfd &   2266 &            ferre \\
de1 &       vad &   1003 &      Schmidtke14 \\
de2 &        va &   2902 &             Vo09 \\
de3 &        va &   1000 &         Kanske10 \\
de4 &     jasfd &   1958 &  Briesemeister12 \\
pl1 &       vad &   4905 &            Imbir \\
pl2 &        va &   2902 &         Riegel15 \\
pl3 &     jasfd &   2902 &          wierzba \\
zh1 &        va &   2794 &             yu16 \\
zh2 &        va &   1100 &            yao16 \\
it  &       vad &   1[...]
{'fraction_non_alphanumeric': 0.11594202898550725, 'fraction_numerical': 0.06262939958592133, 'mean_word_length': 1.7148876404494382, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '122745', 'license': 'MIT', 'n_tokens_mistral': 720, 'n_tokens_neox': 650, 'n_words': 175, 'path': 'memolon/analyses/overview-gold-lexica.ipynb', 'repo_name': 'JULIELab/MEmoLon', 'size': '1932'}