<jupyter_start><jupyter_text>Introspecting State Level WebUI output<jupyter_code>import os
import pandas as pd
import datetime
import seaborn as sns
import matplotlib.pyplot as plt
from pyseir import load_data

# Global settings for the notebook.
base_path_new = '/Users/ecarlson/covid-data-model/output/web_ui'
base_path_old = '/Users/ecarlson/covid-data-model/output/web_ui_master'
comparison_cols = [(8, 'current_icu'), (10, 'all_infected'), (11, 'dead'), (12, 'beds')]

for state in ('AR',):
    plt.figure(figsize=(16, 16))
    plt.title(state)
    
    for i_plt, (col, compartment) in enumerate(comparison_cols):                     
        plt.subplot(2, 2, i_plt + 1)
        for i in range(4):
            color = list(plt.rcParams['axes.prop_cycle'])[i]['color']
            df_new = pd.read_json(os.path.join(base_path_new, f'state/{state}.{i}.json'), convert_dates=[1])                      
            plt.plot(df_new[1] - datetime.timedelta(days=1), df_new[col], label=f'New {i}', color=color)

            df_old = pd.read_json(os.path.join(base_path_old, f'state/{state}.{i}.json'), convert_dates=[1])
            plt.plot(df_old[1], df_old[col], label=f'Old {i}', linestyle='--', color=color)
            plt.legend()

        plt.xlim(datetime.datetime.strptime('3/15/2020', '%m/%d/%Y'), datetime.datetime.strptime('7/15/2020', '%m/%d/%Y'))

        _ = plt.xticks(rotation=30)
        plt.grid(True, which='both', alpha=.5)
        plt.title(compartment)
        plt.yscale('log')
        
        # Different scale options can be helpful.
        #plt.yscale('linear')
        #plt.ylim(0,400)
        #plt.ylim(1e3, None)
        #plt.xlim(datetime.datetime.strptime('4/1/2020', '%m/%d/%Y'), datetime.datetime.strptime('4/30/2020', '%m/%d/%Y'))<jupyter_output><empty_output><jupyter_text>Introspecting County Level Results<jupyter_code>import pandas as pd
import datetime
import seaborn as sns
import os
import matplotlib.pyplot as plt
from pyseir import load_data


for state in ('CA',):
    for fips in ['06037']:
        plt.figure(figsize=(16, 16))
        plt.title(state)
        for i_plt, (col, compartment) in enumerate([(9, 'all_hospitalized'), (13, 'cumulative_infected'), (11, 'dead'), (12, 'beds')]):                     

            plt.subplot(2, 2, i_plt + 1)

            for i in range(4):
                color = list(plt.rcParams['axes.prop_cycle'])[i]['color']

            
                df_pyseir = pd.read_json(os.path.join(base_path_new, f'county/{state}.{fips}.{i}.json'), convert_dates=[1])
                plt.plot(df_pyseir[1] - datetime.timedelta(days=1), df_pyseir[col], label=f'New {i}', color=color)
                
                path = os.path.join(base_path_old, f'county/{state}.{fips}.{i}.json')
                if os.path.exists(path):
                    df_can = pd.read_json(path, convert_dates=[1])
                    plt.plot(df_can[1], df_can[col], label=f'Old {i}', linestyle='--', color=color)
                else:
                    print(f'Could not find {path}')
                plt.legend()

            if compartment == 'all_infected':
                ref_time = datetime.datetime.strptime('3/1/2020', '%m/%d/%Y')
                t_list, cases, deaths = load_data.load_new_case_data_by_fips(fips, t0=datetime.datetime.strptime('3/1/2020', '%m/%d/%Y'))
                dates = [ref_time + datetime.timedelta(days=t) for t in t_list]
                cases = cases.cumsum()
                plt.scatter(dates, cases)

            plt.xlim(datetime.datetime.strptime('3/15/2020', '%m/%d/%Y'), datetime.datetime.strptime('7/15/2020', '%m/%d/%Y'))

            _ = plt.xticks(rotation=30)
            plt.grid(True, which='both', alpha=.5)
            plt.title(compartment)
            plt.yscale('log')

        #plt.yscale('linear')
        plt.ylim(1e1)
        #plt.ylim(1e3, None)
        #plt.xlim(datetime.datetime.strptime('4/1/2020', '%m/%d/%Y'), datetime.datetime.strptime('4/30/2020', '%m/%d/%Y'))<jupyter_output><empty_output><jupyter_text>Checking a state's counties<jupyter_code>outputs = []

from pyseir.load_data import load_county_metadata
import pandas as pd
import us

county_metadata = load_county_metadata()

state = 'CA'
state_obj = us.states.lookup(state)
fips_list = county_metadata[county_metadata.state==state_obj.name].fips


for fips in fips_list:
    for i in range(4):
        county_output = {'fips': fips}
        
        try:
            path = os.path.join(base_path_new, f'county/{state_obj.abbr}.{fips}.{i}.json')
            df_new = pd.read_json(path, convert_dates=[1])
            for col, compartment in comparison_cols:
                county_output[f'{compartment}_new'] = df_new[col].max()
        except:  
            print('Did not find new file at path', path)
        
        try:
            path = os.path.join(base_path_old, f'county/{state_obj.abbr}.{fips}.{i}.json')
            df_old = pd.read_json(path, convert_dates=[1])
            for col, compartment in comparison_cols:
                county_output[f'{compartment}_old'] = df_old[col].max()
        except:  
            print('Did not find old file at path', path)
        
        county_output['intervention'] = i
        outputs.append(county_output)

        
df = pd.DataFrame(outputs)    
df[df.current_icu_new.notnull()].head(50)
plt.figure(figsize=(13, 13))


for i, (_, compartment) in enumerate(comparison_cols):
    plt.subplot(2, 2, i+1)
    sns.scatterplot(x=f'{compartment}_new', y=f'{compartment}_old', hue='intervention', data=df, palette='CMRmap', alpha=.4)
    _range = 1.2 * df[f'{compartment}_new'].max()
    plt.title(compartment)
    plt.plot([1, _range], [1, _range])
    plt.legend()
    plt.yscale('log')
    #plt.xlim(1e0)

    plt.xscale('log')
    plt.grid(True)

import us
outputs = []

for state in [state for state in us.STATES]:
    for i in range(4):
        county_output = {'state': state.name}
        
        path = os.path.join(base_path_new, f'state/{state.abbr}.{i}.json')
        if not os.path.exists(path):
            continue
            
        df_new = pd.read_json(path, convert_dates=[1])
        for col, compartment in comparison_cols:
            county_output[f'{compartment}_new'] = df_new[col].max()
        
        
        path = os.path.join(base_path_old, f'state/{state.abbr}.{i}.json')
        if not os.path.exists(path):
            continue
            
        df_old = pd.read_json(path, convert_dates=[1])
        for col, compartment in comparison_cols:
            county_output[f'{compartment}_old'] = df_old[col].max()

        county_output['intervention'] = i
        outputs.append(county_output)

df = pd.DataFrame(outputs)
plt.figure(figsize=(13, 13))

for i, compartment in enumerate([col[1] for col in comparison_cols]):
    plt.subplot(2, 2, i+1)
    
    sns.scatterplot(x=f'{compartment}_new', y=f'{compartment}_old', hue='intervention', data=df, palette='Set2', alpha=1)

    _range = 1.2 * df[f'{compartment}_new'].max()
    plt.title(compartment)
    plt.plot([1, _range], [1, _range])
    plt.legend()
    plt.yscale('log')
    plt.xscale('log')
    plt.grid(True)

    
    
plt.figure(figsize=(13, 13))

for i, compartment in enumerate([col[1] for col in comparison_cols]):

    plt.subplot(2, 2, i+1)
    for i in range(4):
        df_sample = df[df.intervention == i]
            
        sns.distplot(df_sample[f'{compartment}_new'] / df_sample[f'{compartment}_old'], 
                     label=str(i), hist_kws=dict(cumulative=True, density=True, histtype='step', linewidth=4), 
                     bins=100,
                     kde=False, )
    plt.title(f'Peak Values New / Old: {compartment}')

    plt.legend()
    plt.xlabel('New / Old Counts')
    plt.ylabel('CDF')
    plt.grid(True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1567899654422117, 'fraction_numerical': 0.022654550108793036, 'mean_word_length': 2.6293543892243383, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '253788', 'license': 'MIT', 'n_tokens_mistral': 2894, 'n_tokens_neox': 2715, 'n_words': 480, 'path': 'notebooks/webui_diff.ipynb', 'repo_name': 'chacreton190/covid-data-model', 'size': '7813'}