<jupyter_start><jupyter_text>Collection size<jupyter_code>pprint(list(zip(data_source_names, num_docs)))<jupyter_output>[('Colonia', 99),
 ('SDHK_Latin', 7572),
 ('SDHK_Swedish', 3086),
 ('SemEval2015', 3370),
 ('velux_dipl_danish', 98),
 ('velux_dipl_latin', 358),
 ('velux_facs_danish', 98),
 ('velux_facs_latin', 358)]<jupyter_text>Type-to-token ratio<jupyter_code>pprint(list(zip(data_source_names, ttrs)))<jupyter_output>[('Colonia', 0.03198154829552504),
 ('SDHK_Latin', 0.06206886394765135),
 ('SDHK_Swedish', 0.10999692488328534),
 ('SemEval2015', 0.11403727637915118),
 ('velux_dipl_danish', 0.19768507354714251),
 ('velux_dipl_latin', 0.10083213336287107),
 ('velux_facs_danish', 0.2915425381793095),
 ('velux_facs_latin', 0.21909677692870239)]<jupyter_text>Document length<jupyter_code>from scipy import stats
descriptions = pd.DataFrame()

for name, lengths in zip(data_source_names, doc_lengths):
    descr = pd.DataFrame({name:lengths}).describe().astype(int)
    descriptions = pd.concat([descriptions, descr], axis=1)
descriptions
plt.figure(figsize=(10,10))
plt.boxplot(doc_lengths[1:], labels=data_source_names[1:])
plt.show()
plt.figure(figsize=(10,10))
plt.boxplot(doc_lengths[0], labels=[data_source_names[0]])
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17059748427672955, 'fraction_numerical': 0.14779874213836477, 'mean_word_length': 10.678899082568808, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '625549', 'license': 'MIT', 'n_tokens_mistral': 660, 'n_tokens_neox': 529, 'n_words': 62, 'path': '01-Document_statistics.ipynb', 'repo_name': 'fredrikwahlberg/nodalida21', 'size': '1272'}