<jupyter_start><jupyter_text>Notebook IntentionsThis simple notebook tests that the RespondentHistoryGenerator is working as intended. At the time of generation only 2011-2018 data was accessible.<jupyter_code>import os
from os.path import expanduser

import sys
sys.path.append(os.path.join(expanduser("~"), "meps", "meps_dev"))

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "meps_db.settings")
os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"

import django
django.setup();

from meps_db.components.reference import DATA_FILES_YEARS
from meps_db.processors import RespondentHistoryGenerator
for year in DATA_FILES_YEARS[:8]:
    rhg = RespondentHistoryGenerator(years=[year])
    respondent_history = rhg.run()
    print(f"{year} - {len(respondent_history[year].keys()) :,} Total Respondents")

ex_resp = list(respondent_history[2011].keys())[20]
respondent_history[2011][ex_resp]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11366120218579236, 'fraction_numerical': 0.020765027322404372, 'mean_word_length': 7.723809523809524, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '61197', 'license': 'MIT', 'n_tokens_mistral': 324, 'n_tokens_neox': 296, 'n_words': 62, 'path': 'sandbox/2021_03_test_history_generator/troubleshoot_history_generator.ipynb', 'repo_name': 'explore-meps/meps_analytics', 'size': '915'}