<jupyter_start><jupyter_text>Load Data<jupyter_code>with open("../data/BioASQ-training5b/BioASQ-trainingDataset5b.json") as f:
    data = json.load(f)
len(data["questions"])
questions = [q for q in data["questions"] if q["type"] in ["factoid", "list"]]
len(questions)<jupyter_output><empty_output><jupyter_text>Analysis<jupyter_code>snippets = [q["snippets"] for q in questions]
snippet_counts = [len(s) for s in snippets]
min(snippet_counts)
max(snippet_counts)
sum(snippet_counts) / len(snippet_counts)
snippet_texts = [s["text"] for q_snippets in snippets for s in q_snippets]
snippet_lengths = [len(tokenizer.tokenize(s)) for s in snippet_texts]
min(snippet_lengths)
max(snippet_lengths)
sum(snippet_lengths) / len(snippet_lengths)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13856209150326798, 'fraction_numerical': 0.00261437908496732, 'mean_word_length': 8.69620253164557, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '55333', 'license': 'MIT', 'n_tokens_mistral': 313, 'n_tokens_neox': 298, 'n_words': 55, 'path': '.ipynb_checkpoints/stats-checkpoint.ipynb', 'repo_name': 'chamikasudusinghe/bioasq', 'size': '765'}