<jupyter_start><jupyter_text>"COVID-19 Plots"> "Interactive COVID Plots"- toc: false- branch: master- badges: true- comments: true- categories: [covid-19, jupyter]- hide: false<jupyter_code>#hide
import os
import pandas as pd
from math import ceil
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.font_manager as font_manager
import matplotlib.ticker as ticker
#pd.show_versions(as_json=False)

threshold = 1000
#hide
# Load Data
def load_raw(fn):
    datapath = os.path.join(r'c:\github\covid-19\csse_covid_19_data\csse_covid_19_time_series', fn)

    #datapath = '../csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_global.csv'
    
    raw = pd.read_csv(datapath)
    del raw['Lat']
    del raw['Long']
    raw= raw.replace('Mainland China','China')
    return raw

raw = load_raw('time_series_covid19_confirmed_global.csv')
raw_deaths = load_raw('time_series_covid19_deaths_global.csv')

raw

#hide
# Shift all the series
def shift(df, threshold=1000):
    maxDate = max(df.index)    
    for c in df.columns:
        for i,a in enumerate(df[c]):        
            if i==0 and a>threshold:
                df[c]=df[c].shift(4) # hack to push out china
                break
            if a>=threshold:
                df[c]=df[c].shift(-i)            
                break
    df=df.reset_index(drop=True)
    #del df['China']
    return df

#hide
import altair as alt
from math import pow


def plotdataframe(df, logy=False, legend=True):
    plt.rcParams["figure.figsize"] = [10, 10]
    fig, ax = plt.subplots()
    markers = [item for sublist in ['oxds+*']*ceil(len(df.columns)/5) for item in sublist]    
    for c,m in zip(df,markers):
        df[c].plot(logy=logy, marker=m, fontsize=12,ax=ax, linewidth=2.5)

    font = font_manager.FontProperties(family='sans-serif',
                                   weight='normal',
                                   style='normal', size=12)
    legend and ax.legend(prop=font)

    return ax

def plotdataframe_point(df, logy=False, legend=True, legend_loc='best', title=None):
    return alt.Chart(df)
    if False:
        plt.rcParams["figure.figsize"] = [10, 10]
        fig, ax = plt.subplots()
        markers = [item for sublist in ['oxds+*']*ceil(len(df.columns)/5) for item in sublist]    
        lines = []
        legendcols = []
        maxX = max([max([x for c in df for (x,y) in zip(df.index,df[c]) if y>0])])
        #print(maxX)
        for c,m in zip(df,markers):
            pts = list([x for (x,y) in zip(df.index,df[c]) if y>0])
            ys = list([x for x in df[c] if x>0])
            if len(pts)<=1:
                continue
            legendcols.append(c)
            df[c].plot(logy=logy, marker=m, fontsize=12,ax=ax, linewidth=1, color='#999999')

            offset = (10,0)
            #print([pts[0],pts[-1]],[ys[0],ys[-1]])
            l,=ax.plot([pts[0],pts[-1]],[ys[0],ys[-1]], linewidth=2.5, marker=m)
            lines.append(l)
            growth = round((pow(ys[-1]/ys[0],1.0/(pts[-1]-pts[0]))-1)*100)

            alignment = 'left'
            if pts[-1]>0.9*maxX:
                offset = (0,5)
                alignment = 'center'
            ax.annotate('{} ({}%)'.format(c,growth), xy=(pts[-1],ys[-1]), xytext=offset, textcoords='offset points', ha=alignment, size=12)

        if logy:
            ax.get_yaxis().set_major_formatter(ticker.FuncFormatter(lambda x, p: format(int(x), ',')))

        font = font_manager.FontProperties(family='sans-serif',
                                       weight='normal',
                                       style='normal', size=12)
        legend and ax.legend(lines,legendcols,prop=font, loc=legend_loc)

        title and ax.set_title(title, size=14)
        return ax

def save_figure(maxDate, filename):
    savedir = os.path.join('data',maxDate.strftime('%Y%m%d'))
    if not os.path.isdir(savedir):
        os.makedirs(savedir)
    plt.savefig(os.path.join(savedir,filename))

#hide_input
df = raw.groupby('Country/Region').sum()
df['Country/Region']=df.index
df=df.melt(id_vars='Country/Region',var_name='date', value_name='Confirmed')
df['date']=pd.to_datetime(df['date'])
df = df.sort_values('date')

wpp = pd.read_csv('c:/github/covid-19/wpp/WPP2019_TotalPopulationBySex.csv')
wpp=wpp[(wpp['Time']==2020) & (wpp['Variant']=='Medium')]
wpp['Location'] = wpp['Location'].replace('Iran (Islamic Republic of)','Iran')
wpp['Location'] = wpp['Location'].replace('United States of America', 'US')
wpp['Location'] = wpp['Location'].replace('Republic of Korea', 'Korea, South')
wpp['Location'] = wpp['Location'].replace('Russian Federation', 'Russia')
#wpp = dict(zip(wpp['Location'],wpp['PopTotal']))
df = df.merge(wpp, left_on='Country/Region', right_on='Location')
df['ConfirmedNorm'] = df['Confirmed'] / df['PopTotal']

# df is in tidy format, so let's keep this.
tidy = df

countries = df.groupby('Country/Region').max().sort_values('Confirmed', ascending=False).head(10)
countries=list(countries.index)

#print(df[df['Country/Region']=='US'])
df = df[df['Country/Region'].apply(lambda x: x in countries)]
df = df[df['Confirmed']>0]

alt.Chart(df).mark_line().encode(x='date',y=alt.Y('Confirmed', scale=alt.Scale(type='log')), color='Country/Region').properties(
    width=600,
    height=300
)
#hide
tidy
#hide_input
vals_1k = tidy[tidy['Confirmed'].apply(lambda x: x>1000)].groupby('Country/Region').apply(lambda df:pd.Series({'Confirmed':df['Confirmed'].min()}))
#vals_1k['Country/Region']=vals_1k.index
vals_1k.reset_index(level=0, inplace=True)
vals_1k
dates_1k = tidy.merge(vals_1k, on=['Country/Region','Confirmed'])
dates_1k['date_1k']=dates_1k['date']
del dates_1k['date']
shifted = tidy.merge(dates_1k, on='Country/Region', how='left')
shifted['Confirmed']=shifted['Confirmed_x']
shifted['ConfirmedNorm']=shifted['ConfirmedNorm_x']
shifted = shifted[shifted['date']>=shifted['date_1k']]
shifted['days'] = shifted['date'] - shifted['date_1k']
shifted['days'] = shifted['days'].apply(lambda x: x.days)
shifted
plt_data = shifted[shifted['Country/Region'].apply(lambda x: x in countries and x!='China')]

alt.Chart(plt_data).mark_line().encode(x='days',y=alt.Y('Confirmed', scale=alt.Scale(type='log')), color='Country/Region').properties(
    width=600,
    height=300
)
#hide_input
normCountries = tidy[tidy['Confirmed']>1000].groupby('Country/Region').max().sort_values('ConfirmedNorm', ascending=False).head(10)
normCountries=list(normCountries.index)
plt_data = shifted[shifted['Country/Region'].apply(lambda x: x in normCountries)]

alt.Chart(plt_data).mark_line().encode(x='days',y=alt.Y('ConfirmedNorm', scale=alt.Scale(type='log')), color='Country/Region').properties(
    width=600,
    height=300
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1749703440094899, 'fraction_numerical': 0.022390272835112694, 'mean_word_length': 3.7701555869872703, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '177732', 'license': 'Apache-2.0', 'n_tokens_mistral': 2506, 'n_tokens_neox': 2369, 'n_words': 420, 'path': '_notebooks/2020-04-10-COVID-19 Plots.ipynb', 'repo_name': 'simra/covid19_fastpages', 'size': '6744'}