<jupyter_start><jupyter_text>Testing vaccinated and unvaccinated population outcomes **Note:** Vaccination scenarios operate under the assumption that the delta variant is dominant.<jupyter_code>import pandas as pd
import numpy as np
from os.path import join
import os
import json
import seaborn as sns
import matplotlib.pyplot as plt

from scseirx import analysis_functions as af

import data_creation_functions as dcf

# parallelisation functionality
from multiprocess import Pool
import psutil
from tqdm import tqdm<jupyter_output><empty_output><jupyter_text>Vaccine effectiveness against Delta (see [Pouwels et al. 2021](https://www.ndm.ox.ac.uk/files/coronavirus/covid-19-infection-survey/finalfinalcombinedve20210816.pdf))* 67% effectiveness against infection 14 days after second dose of Astra Zeneca (ChAdOx1)* 80% effectiveness against infection 21 days after second dose of Biontech Pfizer (BNT162b2)* Viral load of vaccinated infected people seems to be similar to unvaccinated infected people ([CDC 2021](https://www.cdc.gov/mmwr/volumes/70/wr/mm7031e2.htm)).* Vaccine effectiveness seems to drop over time ([Pouwels et al. 2021](https://www.ndm.ox.ac.uk/files/coronavirus/covid-19-infection-survey/finalfinalcombinedve20210816.pdf)) (78% effectiveness against high viral load after 90 days for Biontech Pfizer and 61% effectiveness against high viral load after 90 days for Astra Zeneca). Go for 60% effectiveness against infection and 0% effectiveness against further gransmission to be conservative.   Simulation settings<jupyter_code>with open('params/vaccinations_measures.json', 'r') as fp:
    measures = json.load(fp)
with open('params/vaccinations_simulation_parameters.json', 'r') as fp:
    simulation_params = json.load(fp)
with open('params/vaccinations_school_characteristics.json', 'r') as fp:
    school_characteristics = json.load(fp)<jupyter_output><empty_output><jupyter_text>Simulation function<jupyter_code>def run(params):
    '''
    Runs an ensemble of simulations and collects observable statistics. To be 
    run in parallel on many workers. Note: I/O paths and the number of runs per 
    ensemble hare hard coded here, because I only want to pass the parameter 
    values that are being screened in the simulation run to the function via the
    parallel processing interface.
    
    Parameters:
    -----------
    param_list : iterable
        Iterable that contains the values for the parameters test_type, 
        index_case, e_screen_range and r_screen_range that are passed to the
        simulation.
        
    Returns:
    --------
    row : dictionary
        Dictionary of the ensemble statistics of the observables.
    '''    

    # extract the simulation parameters from the parameter list
    N_runs, school_type, index_case, s_screen_interval, t_screen_interval,\
        student_mask, teacher_mask, half_classes, ventilation_mod,\
        student_vaccination_ratio, teacher_vaccination_ratio, \
        family_member_vaccination_ratio = params
    
    try:
        os.mkdir(join(dst, school_type))
    except FileExistsError:
        pass
    
    ttype = 'same_day_antigen'

    # run the ensemble with the given parameter combination and school type
    ensmbl_results = dcf.run_ensemble(N_runs, school_type, measures,\
            simulation_params, school_characteristics, contact_network_src,\
            dst, index_case, ttype=ttype, s_screen_interval=s_screen_interval,
            t_screen_interval=t_screen_interval, student_mask=student_mask,
            teacher_mask=teacher_mask, half_classes=half_classes,
            ventilation_mod=ventilation_mod, 
            student_vaccination_ratio=student_vaccination_ratio,
            teacher_vaccination_ratio=teacher_vaccination_ratio,
            family_member_vaccination_ratio=family_member_vaccination_ratio)
    
    ensmbl_results['school_type'] = school_type
    ensmbl_results['index_case'] = index_case
    ensmbl_results['test_type'] = ttype
    ensmbl_results['student_screen_interval'] = s_screen_interval
    ensmbl_results['teacher_screen_interval'] = t_screen_interval
    ensmbl_results['student_mask'] = student_mask
    ensmbl_results['teacher_mask'] = teacher_mask
    ensmbl_results['half_classes'] = half_classes
    ensmbl_results['ventilation_mod'] = ventilation_mod
    ensmbl_results['student_vaccination_ratio'] = student_vaccination_ratio
    ensmbl_results['teacher_vaccination_ratio'] = teacher_vaccination_ratio
    ensmbl_results['family_member_vaccination_ratio'] = family_member_vaccination_ratio
    
    return ensmbl_results<jupyter_output><empty_output><jupyter_text>Screening parameters<jupyter_code># number of runs in the ensemble (set to a low number so runs complete quickly
# for demonstration purposes)
N_runs = 500
# different school types (only simulate for primary schools for demonstration
# purposes)
school_types = ['secondary', 'upper_secondary', 'lower_secondary_dc', 
                'lower_secondary', 'primary_dc', 'primary']
# load the other screening parameters from file
screening_params = pd.read_csv(join('screening_params', 'vaccinations.csv'))

params = [(N_runs, st, 
           row['index_case'],
           dcf.format_none_column(row['s_screen_interval']),
           dcf.format_none_column(row['t_screen_interval']),
           row['s_mask'],
           row['t_mask'], 
           row['half_classes'],
           row['ventilation_modification'],
           row['student_vaccination_ratio'],
           row['teacher_vaccination_ratio'],
           row['family_member_vaccination_ratio']) \
           for st in school_types \
           for i, row in screening_params.iterrows()]

print('there are {} different parameter combinations'.format(len(params)))<jupyter_output>there are 192 different parameter combinations<jupyter_text>Run simulation<jupyter_code>%%time
# paths for data I/O
contact_network_src = '../../data/contact_networks/representative_schools'
dst = '../../data/vaccinations/simulation_results'

number_of_cores = psutil.cpu_count(logical=True)
pool = Pool(number_of_cores)

results = pd.DataFrame()
for ensmbl_results in tqdm(pool.imap_unordered(func=run, iterable=params),
                           total=len(params)):
    results = results.append(ensmbl_results, ignore_index=True)

# turn off your parallel workers 
pool.close()
    
results = results.reset_index(drop=True)
index_cols = ['school_type', 'index_case', 'test_type',
              'student_screen_interval', 'teacher_screen_interval',
              'student_mask', 'teacher_mask', 'half_classes',
              'ventilation_mod', 'student_vaccination_ratio',
              'teacher_vaccination_ratio', 'family_member_vaccination_ratio']
other_cols = [c for c in results.columns if c not in index_cols]
results = results[index_cols + other_cols]

results.to_csv(join(dst, 'vaccinations_secondary_{}.csv'\
                   .format(N_runs)), index=False)
results.head(3)<jupyter_output>100%|██████████| 32/32 [2:11:48<00:00, 247.14s/it]<jupyter_text>Aggregate results<jupyter_code>src = '../../data/vaccinations/simulation_results'
dst = src
for stype in school_types:
    print(stype)
    data = dcf.get_data(stype, src)
    data['measure'] = np.nan
    dcf.set_individual_measures(data)
    dcf.set_measure_packages(data)
    data.to_csv(join(dst, '{}_combined_ensembles.csv'.format(stype)), index=False)<jupyter_output>secondary
upper_secondary
lower_secondary_dc
lower_secondary
primary_dc
primary<jupyter_text>Extract observables<jupyter_code>dst = '../../data/vaccinations/simulation_results'

turnovers = {'same':0, 'one':1, 'two':2, 'three':3}
bmap = {True:'T', False:'F'}


for stype in school_types:
    print(stype)
    spath_ensmbl = join(dst,'{}'.format(stype))
    
    observables = pd.DataFrame()
    for N_runs, school_type, index_case, s_screen_interval, t_screen_interval,\
        student_mask, teacher_mask, half_classes, ventilation_mod,\
        student_vaccination_ratio, teacher_vaccination_ratio, \
        family_member_vaccination_ratio in params:
        
        ttype = 'same_day_antigen'
        
        turnover, _, test = ttype.split('_')
        turnover = turnovers[turnover]
        
        measure_string = '{}_test-{}_turnover-{}_index-{}_tf-{}_sf-{}_tmask-{}'\
        .format(stype, test, turnover, index_case[0], t_screen_interval,
                s_screen_interval, bmap[teacher_mask]) +\
                '_smask-{}_half-{}_vent-{}_tvacc-{}_svacc-{}'\
        .format(bmap[student_mask], bmap[half_classes], ventilation_mod,
                teacher_vaccination_ratio, student_vaccination_ratio)
        
        ensmbl = pd.read_csv(join(spath_ensmbl, measure_string + '.csv'))
        ensmbl = ensmbl.drop(columns=['Unnamed: 0'])
        
        row = {'test_type':test,
               'turnover':turnover,
               'index_case':index_case,
               'student_screen_interval':s_screen_interval,
               'teacher_screen_interval':t_screen_interval,
               'student_mask':student_mask,
               'teacher_mask':teacher_mask,
               'half_classes':half_classes,
               'ventilation_modification':ventilation_mod}
        
        ensmbl = ensmbl[ensmbl['infected_agents'] > 0]
        for col in ensmbl.columns:
            row.update(af.get_statistics(ensmbl, col))
        observables = observables.append(row, ignore_index=True)

    # calculate the number of tests per day and agent in the school
    observables['N_tests_per_day_per_agent'] = \
    (observables['N_diagnostic_tests_median'] + observables['N_preventive_tests_median']) /\
    observables['duration_median'] / observables['N_school_agents_median']
    
    screen_cols = ['test_type', 'turnover', 'index_case', 'student_screen_interval',
            'teacher_screen_interval', 'student_mask', 'teacher_mask',
            'half_classes', 'ventilation_modification']

    other_cols = [c for c in observables.columns if c not in screen_cols]
    observables = observables[screen_cols + other_cols]
    
    for col in ['infected_teachers_median', 'infected_students_median', 
                        'infected_family_members_median',
                        'infected_teachers_0.90', 'infected_students_0.90', 
                        'infected_family_members_0.90']:
        observables[col] = observables[col].round(0).astype(int)
    
    observables.to_csv(join(dst, '{}_observables'.format(stype) + '.csv'))<jupyter_output>secondary
upper_secondary
lower_secondary_dc
lower_secondary
primary_dc
primary
{'fraction_non_alphanumeric': 0.09684150621265294, 'fraction_numerical': 0.010243763634639098, 'mean_word_length': 3.901906090190609, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 3, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '602096', 'license': 'MIT', 'n_tokens_mistral': 3326, 'n_tokens_neox': 3135, 'n_words': 680, 'path': 'code/vaccinations/vaccinations_data_creation.ipynb', 'repo_name': 'JanaLasser/school_SEIRX', 'size': '10543'}