<jupyter_start><jupyter_text>Check the zenodo files<jupyter_code>import requests

# you need to modify this to your zenodo ID
zenodo_id = 2648592 # old grl data
# zenodo_id = 4739876 # new files...which dont work (yet?)...
zenodo_id = 4739912 # a tiny test dataset Check again tomorrow.
zenodo_id = 812722

# r = requests.get("https://zenodo.org/api/records/%i" % zenodo_id)
r = requests.get("https://sandbox.zenodo.org/api/records/%i" % zenodo_id)

file = open("zenodo_filelist.txt", "w")

for f in r.json()["files"]:
    file.write(f["links"]["self"] + "\n")

file.close()
r.json()['files']
https://zenodo.org/api/files/9adbe6da-8597-4ef4-89f7-a91cff2937e9/busecke_etal_2021_aguadv.tar<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16178521617852162, 'fraction_numerical': 0.06834030683403068, 'mean_word_length': 4.983333333333333, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '344941', 'license': 'MIT', 'n_tokens_mistral': 321, 'n_tokens_neox': 292, 'n_words': 55, 'path': 'notebooks/old/check_zenodo_files.ipynb', 'repo_name': 'jbusecke/busecke_etal_2021_aguadv', 'size': '717'}