<jupyter_start><jupyter_text>Kaggle COVID19 California forecastBased on https://www.kaggle.com/samkennerly/contagion-window-model<jupyter_code>%load_ext autoreload
%autoreload 2
%autosave 0
from datetime import timedelta
from numpy import linspace
from pathlib import Path
from pandas import DataFrame, Series, read_csv
FIGSIZE = (9, 5)
FOLDER = Path("../../data/kaggle/california").resolve()
OUTPUT = FOLDER / 'submission.csv'
TRAIN = FOLDER / 'raw/ca_train.csv'
TEST = FOLDER / 'raw/ca_test.csv'<jupyter_output><empty_output><jupyter_text>get dataLoad training data and forecast dates.<jupyter_code>fid = (
    read_csv(TEST, parse_dates=['Date'])
    .set_index('Date')['ForecastId']
)
fid.tail()
train = (
    read_csv(TRAIN, parse_dates=['Date'])
    .set_index('Date')['ConfirmedCases Fatalities'.split()]
)
train.tail(16)<jupyter_output><empty_output><jupyter_text>modelIn this example, infected persons become immediately contagious, then quarantine themselves a few days later when symptoms appear.<jupyter_code>def contagion_window(delay=4, duration=14, reproduction=2.5, **kwargs):
    window = Series(linspace(1, 0, duration), name='window')
    window *= reproduction / window.sum()
    window[delay:] = 0
    
    return window.rename_axis('days since infection')

plotkw = {'figsize': FIGSIZE }
plotkw['color'] = 'red'
plotkw['title'] = 'expected transmissions'
axes = contagion_window().plot.bar(**plotkw)
def convolved(values, window):
    """ int or float: Discrete convolution of two timeseries. """
    return sum( w * x for w, x in zip(window, reversed(values)) )

def tomorrow(deltas, window, population=40_000_000, vax_rate=0, **kwargs):
    """ int: Predicted number of new cases on next day. """
    sfrac = (1 - vax_rate - sum(deltas) / population )
    
    return int(sfrac * convolved(deltas, window))

def predict(train, dates, confirm_rate=0.2, **kwargs):
    """ DataFrame: Predicted future cases and fatalities. """

    aday = timedelta(days=1)
    data = DataFrame(index=dates)
    future = data.index.difference(train.index)
    window = contagion_window(**kwargs)
    duration = len(window)

    # Backfill data from training-testing overlap
    data['confirmed'] = train['ConfirmedCases']
    data['deceased'] = train['Fatalities']
    
    # Estimate infection counts and real mortality rate
    data['confirmed'] = train['ConfirmedCases']
    data['exposed'] = (train['ConfirmedCases'] / confirm_rate)
    mortality = data['deceased'].sum() / data['exposed'].sum()
    print("Estimated mortality:", round(100 * mortality, 2), "%")
 
    # Predict new cases
    data['new'] = data['exposed'].diff()
    data.at[dates.min(), 'new'] = data['exposed'][0]
    for t in future:
        deltas = data.loc[:t-aday, 'new']
        data.at[t, 'new'] = tomorrow(deltas, window, **kwargs)
        
    # Calculate cumulative totals
    loc = data.loc
    loc[future, 'exposed'] = data['new'].cumsum()
    loc[future, 'deceased'] = loc[future, 'exposed'] * mortality
    loc[future, 'confirmed'] = loc[future, 'exposed'] * confirm_rate

    return data.sort_index(axis=1).astype(int)
    
data = predict(train, fid.index)
axes = data.plot.line(color='bkgr', figsize=FIGSIZE, logy=True)
data.tail()<jupyter_output>Estimated mortality: 0.36 %<jupyter_text>submit results<jupyter_code>submission = (
    DataFrame(index = data.index)
    .assign(ForecastId = fid)
    .assign(ConfirmedCases = data['confirmed'])
    .assign(Fatalities = data['deceased'])
)
print("Save", OUTPUT)
submission.to_csv(OUTPUT, index=False)
submission.tail()<jupyter_output>Save /context/data/kaggle/california/submission.csv
{'fraction_non_alphanumeric': 0.14754550468836183, 'fraction_numerical': 0.010479867622724766, 'mean_word_length': 4.702830188679245, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '622268', 'license': 'Apache-2.0', 'n_tokens_mistral': 1256, 'n_tokens_neox': 1192, 'n_words': 265, 'path': 'books/kaggle/ca_forecast.ipynb', 'repo_name': 'samkennerly/covid19', 'size': '3626'}