<jupyter_start><jupyter_text>Imports<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

import os, sys
sys.path.insert(0, os.getcwd()[:os.getcwd().rfind('\\')])
os.chdir(sys.path[0])
from scripts import Patient, Measurement, Simulator, Filter
from scripts import base_rule, sd_rule, arv_rule<jupyter_output><empty_output><jupyter_text>Data<jupyter_code>df = pd.read_csv('data/diary_param.csv', sep='\t')
df.head()
patients = df.patient_id.unique()
print(f'There are {len(patients)} patients.')<jupyter_output>There are 137 patients.<jupyter_text>Experiments  1 patient 1 simulation run<jupyter_code>pat = patients[0]
df_1 = df[df.patient_id == pat]
df_1.to_csv(f'data/patient_{pat}', sep='\t', index=False)
p = Patient(f'data/patient_{pat}')
n_days = 30
sim = Simulator(p)
sim.run_simulation(n_days)
os.remove(f'data/patient_{pat}')  # don't need it anymore
for meas in sim.measurements:
    print(meas.sbp, meas.dbp)  # -1 is a missing measurement
# it's cool that when sbp (dbp) is -1, the map will be -1 :)
map_values = np.array([(meas.sbp + 2 * meas.dbp) / 3 for meas in sim.measurements])  # MAP is the mean arterial pressure
f = Filter([
            base_rule,
            sd_rule,
            arv_rule,
           ])
f.fit(sim.measurements)
results = f.apply(sim.measurements)[:]
def filter_missing(meas_array, apply_array):
    return np.where(meas_array == -1, None, apply_array)  # -1 is a missing measurement
filtered_vals = filter_missing(map_values, map_values)
def plot_meas_series(vals, checks, title=None):
    fig, ax = plt.subplots(figsize=(10,4))
    ax.plot(vals, 'o--', markersize='4')
    ax.plot(np.where(checks == True, None, vals), 'x', markersize='6', label='alerts')
    ax.set_xlabel('measurement number')
    ax.set_ylabel('MAP')
    ax.set_title(title)
    ax.legend();
    ax.spines['top'].set_visible(False)
    ax.spines['left'].set_visible(False)
    ax.spines['right'].set_visible(False)
for checks, rule in zip(results, ['Basic rules', 'Mean-based rule', 'ARV-based rule']):
    plot_meas_series(filtered_vals, filter_missing(map_values, checks), rule)<jupyter_output><empty_output><jupyter_text>1 patient many simulation runs<jupyter_code>pat = 194937  # patients[0]
df_1 = df[df.patient_id == pat]
df_1.to_csv(f'data/patient_{pat}', sep='\t', index=False)
p = Patient(f'data/patient_{pat}')
p.state_prob.missing = .8
pat_meas = [Measurement(s, d) for s, d in zip(p.get_sbp_sequence(), p.get_dbp_sequence())] # fit на изначальном наборе измерений
%%time
n_days = 30
sim = Simulator(p)

sim_runs = []
for _ in range(100):
    sim.run_simulation(n_days)
    sim_runs.append(sim.measurements)
    sim.reset_simulation()
os.remove(f'data/patient_{pat}')  # don't need it anymore
checks = []
f = Filter([
            base_rule,
            sd_rule,
            arv_rule,
           ])
f.fit(pat_meas)
for sim_meas in sim_runs:
    checks.append(dict(zip(['base', 'sd', 'arv'], f.apply(sim_meas))))
base_load, sd_load, arv_load = [], [], []

for i, check in enumerate(checks):
    map_values = np.array([(meas.sbp + 2 * meas.dbp) / 3 for meas in sim_runs[i]])
    meas_cnt = (map_values != -1).sum() + 10e-10  # without missing
    base_load.append((filter_missing(map_values, check['base']) == False).sum() / meas_cnt)
    sd_load.append((filter_missing(map_values, check['sd']) == False).sum() / meas_cnt)
    arv_load.append((filter_missing(map_values, check['arv']) == False).sum() / meas_cnt)
plt.plot(base_load, sd_load, '.')
plt.plot([0,1], [0,1], '-')
plt.xlabel('Base rule')
plt.ylabel('Mean-based rule');
plt.plot(base_load, arv_load, '.')
plt.plot([0,1], [0,1], '-')
plt.xlabel('Base rule')
plt.ylabel('Arv-based rule');<jupyter_output><empty_output><jupyter_text>Many patients many simulation runs<jupyter_code>from collections import defaultdict
from tqdm.notebook import tqdm

n_days = 30
results = defaultdict(dict)

for pat in tqdm(patients):
    df_1 = df[df.patient_id == pat]
    df_1.to_csv(f'data/patient_{pat}', sep='\t', index=False)
    p = Patient(f'data/patient_{pat}')
    pat_meas = [Measurement(s, d) for s, d in zip(p.get_sbp_sequence(), p.get_dbp_sequence())]
    
    sim = Simulator(p)
    sim_runs = []
    for _ in range(100):
        sim.run_simulation(n_days)
        sim_runs.append(sim.measurements)
        sim.reset_simulation()
    os.remove(f'data/patient_{pat}')
    
    checks = []
    f = Filter([
                base_rule,
                sd_rule,
                arv_rule,
               ])
    try:
        f.fit(pat_meas)
        for sim_meas in sim_runs:
            checks.append(dict(zip(['base', 'sd', 'arv'], f.apply(sim_meas))))
    except ValueError:
        continue
        
    base_load, sd_load, arv_load = [], [], []
    for i, check in enumerate(checks):
        map_values = np.array([(meas.sbp + 2 * meas.dbp) / 3 for meas in sim_runs[i]])
        meas_cnt = (map_values != -1).sum() + 10e-10  # without missing
        base_load.append((filter_missing(map_values, check['base']) == False).sum() / meas_cnt)
        sd_load.append((filter_missing(map_values, check['sd']) == False).sum() / meas_cnt)
        arv_load.append((filter_missing(map_values, check['arv']) == False).sum() / meas_cnt)
    
    results[pat] = {'base': base_load, 'sd': sd_load, 'arv': arv_load}
base_load_means = [np.mean(results[p]['base']) for p in results]
sd_load_means = [np.mean(results[p]['sd']) for p in results]
arv_load_means = [np.mean(results[p]['arv']) for p in results]
plt.boxplot([base_load_means, sd_load_means, arv_load_means],
            labels=['base', 'mean-based', 'arv-based'],
            showmeans=True)
plt.text(2.9,1.1,f'{len(base_load_means)} patients');
print(f"Base load mean: {np.mean(base_load_means)}\t ARV load mean: {np.mean(arv_load_means)}")<jupyter_output>Base load mean: 0.691983140843371	 ARV load mean: 0.722486932076201
{'fraction_non_alphanumeric': 0.16115843270868824, 'fraction_numerical': 0.01788756388415673, 'mean_word_length': 4.061206896551724, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '563765', 'license': 'MIT', 'n_tokens_mistral': 2287, 'n_tokens_neox': 2174, 'n_words': 432, 'path': 'notebooks/information_filtering.ipynb', 'repo_name': 'Siella/BP-SIM', 'size': '5870'}