<jupyter_start><jupyter_text>Load data<jupyter_code>df_us_confirmed_case = pd.read_csv('RAW_us_confirmed_cases.csv')
df_us_confirmed_case
df_us_mask_mandate = pd.read_csv('U.S._State_and_Territorial_Public_Mask_Mandates_From_April_10__2020_through_August_15__2021_by_County_by_Day.csv')
df_us_mask_mandate
df_mask_use_by_county = pd.read_csv('mask-use-by-county.csv')
df_mask_use_by_county<jupyter_output><empty_output><jupyter_text>collect data needed and clean up<jupyter_code>state = 'Florida'
state_abbr = 'FL'
county = 'Palm Beach'
FIPS = 12099

# clean up data
# drop unneed columns
df_us_confirmed_case_dropped = df_us_confirmed_case.drop(columns=['UID', 'iso2', 'iso3', 'code3', 'Country_Region', 'Lat', 'Long_', 'Combined_Key'])
df_us_confirmed_case_dropped=df_us_confirmed_case_dropped.fillna(0)
df_us_confirmed_case_dropped['FIPS'] =  df_us_confirmed_case_dropped.FIPS.apply(lambda x:  "{:05d}".format(int(float(x))))
df_us_confirmed_case_dropped
# Melt the confirm case data so each row repreasent confirmed case each day

id_vars = [
    'Province_State',
    'Admin2',
    'FIPS',
]

df_us_confirmed_case_transformed =pd.melt(df_us_confirmed_case_dropped, id_vars = id_vars, var_name ='date', value_name = "cases")
df_us_confirmed_case_transformed['date'] = pd.to_datetime(df_us_confirmed_case_transformed['date'])
df_us_confirmed_case_transformed
df_county_population = pd.read_csv('US County Assignments - County Assignments.csv',  thousands=',')
df_county_population = df_county_population[['County', 'State', 'Population_2020_Census']]
df_county_population.fillna(0)
#df_county_population['Population_2020_Census'] = df_county_population['Population_2020_Census'].astype(int)
df_county_population
df_us_confirmed_case_transformed_with_pop = df_us_confirmed_case_transformed.merge(df_county_population, left_on='Admin2', right_on='County')
df_us_confirmed_case_transformed_with_pop

df_palm_beach_confirmed_case_transformed = df_us_confirmed_case_transformed[(df_us_confirmed_case_transformed['Admin2'] == county) & (df_us_confirmed_case_transformed['Province_State'] == state)].reset_index(drop=True)
df_palm_beach_confirmed_case_transformed
# create FIPS column for easier join
df_us_mask_mandate['FIPS_State'] =  df_us_mask_mandate.FIPS_State.apply(lambda x:  "{:02d}".format(int(x)))
df_us_mask_mandate['FIPS_County'] =  df_us_mask_mandate.FIPS_County.apply(lambda x:  "{:03d}".format(int(x)))
df_us_mask_mandate['FIPS'] = df_us_mask_mandate['FIPS_State'] +  df_us_mask_mandate['FIPS_County'] 
df_us_mask_mandate

# drop unneed columns
df_us_mask_mandate_dropped = df_us_mask_mandate.drop(columns=['FIPS_State', 'FIPS_County', 'order_code', 'Source_of_Action', 'URL', 'Citation'])
df_us_mask_mandate_dropped

df_palm_beach_mask_mandate = df_us_mask_mandate_dropped[df_us_mask_mandate['County_Name'] == 'Palm Beach County'].reset_index(drop =True)
df_palm_beach_mask_mandate
df_palm_beach_mask_mandate.Face_Masks_Required_in_Public.unique()
df_mask_use_palm_beach = df_mask_use_by_county[df_mask_use_by_county['COUNTYFP'] == FIPS].reset_index(drop =True)
df_mask_use_palm_beach<jupyter_output><empty_output><jupyter_text>Palm Beach has no mask mandate<jupyter_code>df_mask_use_palm_beach_transformed = df_mask_use_palm_beach.drop(columns=['COUNTYFP'])
df_mask_use_palm_beach_transformed =pd.melt(df_mask_use_palm_beach_transformed, var_name ='Response', value_name = "Proportion")
df_mask_use_palm_beach_transformed<jupyter_output><empty_output><jupyter_text>Visualize<jupyter_code>import matplotlib.pyplot as plt
import matplotlib.lines as lines

from datetime import datetime
from matplotlib.dates import date2num

fig, ax = plt.subplots(figsize=(15,10))
ax.bar(df_mask_use_palm_beach_transformed['Response'], df_mask_use_palm_beach_transformed['Proportion'], color  ='C0')

ax.set_title("Estimated Prevalence of Mask Wearing in Palm Beach Country, FL")
ax.set_ylabel("Proportion")

plt.show()
# accumulated covid cases
fig, ax = plt.subplots(figsize=(15,10))
ax.plot(df_palm_beach_confirmed_case_transformed['date'], df_palm_beach_confirmed_case_transformed['cases'], color='C0')

ax.set_title("Accumulate Covid Cases, Palm Beach Country, FL")
ax.set_xlabel("Date")
ax.set_ylabel("Number of Covid Cases")

plt.show()
# daily new covid cases
df_palm_beach_confirmed_case_transformed['new_cases'] = df_palm_beach_confirmed_case_transformed['cases'].diff()
df_palm_beach_confirmed_case_transformed['new_cases_moving_average_7_days'] = df_palm_beach_confirmed_case_transformed['new_cases'].rolling(window=7).mean().round()
df_palm_beach_confirmed_case_transformed
# daily new covid cases
fig, ax = plt.subplots(figsize=(15,10))
ax.plot(df_palm_beach_confirmed_case_transformed['date'], df_palm_beach_confirmed_case_transformed['new_cases'], color='C0')
ax.plot(df_palm_beach_confirmed_case_transformed['date'], df_palm_beach_confirmed_case_transformed['new_cases_moving_average_7_days'], color='C1')


line0 = lines.Line2D([0], [0], label='Daily New COVID Cases', color='C0')
line1 = lines.Line2D([0], [0], label='7 days Moving average New COVID Cases', color='C1')
plt.legend(handles=[line0,line1])

ax.set_title("Daily New Covid Cases & 7 days average, Palm Beach Country, FL")
ax.set_xlabel("Date")
ax.set_ylabel("Number of Covid Cases")

plt.show()
# infection rate = daily new case / population
population = 1492191
df_palm_beach_confirmed_case_transformed['daily_infection_rate'] = df_palm_beach_confirmed_case_transformed['new_cases'].apply(lambda x: x * 1.0 / population)
df_palm_beach_confirmed_case_transformed['daily_infection_rate_new_cases_moving_average_7_days'] = df_palm_beach_confirmed_case_transformed['new_cases_moving_average_7_days'].apply(lambda x: x * 1.0 / population)

df_palm_beach_confirmed_case_transformed
# daily new covid cases
fig, ax = plt.subplots(figsize=(15,10))
#ax.plot(df_palm_beach_confirmed_case_transformed['date'], df_palm_beach_confirmed_case_transformed['daily_infection_rate'], color='C0')
ax.plot(df_palm_beach_confirmed_case_transformed['date'], df_palm_beach_confirmed_case_transformed['daily_infection_rate_new_cases_moving_average_7_days'], color='C1')



#line0 = lines.Line2D([0], [0], label='Daily Infection rate', color='C0')
line1 = lines.Line2D([0], [0], label='7 days Moving average infection rate', color='C1')

plt.legend(handles=[
    #line0,
    line1])

ax.set_title("7 days moving average infection rate, Palm Beach Country, FL (Daily new cases / Population)")
ax.set_xlabel("Date")
ax.set_ylabel("Infection Rate")

plt.show()<jupyter_output><empty_output><jupyter_text>since Palm Beach county doesn't have mandate mask policy, try to find<jupyter_code># 

df_palm_beach_confirmed_case_transformed['daily_infection_rate_diff'] =df_palm_beach_confirmed_case_transformed['daily_infection_rate'].diff()
df_palm_beach_confirmed_case_transformed['daily_infection_rate_new_cases_moving_average_7_days_diff'] =df_palm_beach_confirmed_case_transformed['daily_infection_rate_new_cases_moving_average_7_days'].diff()

df_palm_beach_confirmed_case_transformed
fig, ax = plt.subplots(figsize=(15,10))
#ax.plot(df_palm_beach_confirmed_case_transformed['date'], df_palm_beach_confirmed_case_transformed['daily_infection_rate_diff'], color='C0')
ax.plot(df_palm_beach_confirmed_case_transformed['date'], df_palm_beach_confirmed_case_transformed['daily_infection_rate_new_cases_moving_average_7_days_diff'], color='C1')



#line0 = lines.Line2D([0], [0], label='Daily Infection rate Diff', color='C0')
line1 = lines.Line2D([0], [0], label='7 days Moving average infection rate Diff', color='C1')

plt.legend(handles=[
    #line0,
    line1])

ax.set_title("7 days moving average infection rate diff, Palm Beach Country, FL")
ax.set_xlabel("Date")
ax.set_ylabel("Infection Rate Diff")

plt.show()<jupyter_output><empty_output><jupyter_text>Infection rate has no much change, specially using the 7 days moving average  Try to find a different county and do compare?find a similar county which has similar always waring masks, TX	Val Verde County<jupyter_code>FIPS_other = 51177
df_mask_use_other = df_mask_use_by_county[df_mask_use_by_county['COUNTYFP'] == FIPS_other].reset_index(drop =True)
df_mask_use_other

df_val_verde_mask_mandate = df_us_mask_mandate_dropped[df_us_mask_mandate['FIPS'] == "{:05d}".format(FIPS_other)].reset_index(drop =True)
df_val_verde_mask_mandate
df_val_verde_mask_mandate.Face_Masks_Required_in_Public.unique()

df_val_verde_mask_mandate_yes = df_val_verde_mask_mandate[df_val_verde_mask_mandate['Face_Masks_Required_in_Public'] == 'Yes'].reset_index(drop =True)
df_val_verde_mask_mandate_yes

df_val_verde_mask_mandate = df_us_mask_mandate_dropped[df_us_mask_mandate['FIPS'] == "{:05d}".format(FIPS_other)].reset_index(drop =True)
df_val_verde_mask_mandate

df_spotsylvania_confirmed_case_transformed = df_us_confirmed_case_transformed[(df_us_confirmed_case_transformed['FIPS'] ==   "{:05d}".format(FIPS_other))].reset_index(drop=True)
df_spotsylvania_confirmed_case_transformed
Population_other = 136215
df_spotsylvania_confirmed_case_transformed['new_cases'] = df_spotsylvania_confirmed_case_transformed['cases'].diff()
df_spotsylvania_confirmed_case_transformed['new_cases_moving_average_7_days'] = df_spotsylvania_confirmed_case_transformed['new_cases'].rolling(window=7).mean().round()
df_spotsylvania_confirmed_case_transformed['daily_infection_rate'] = df_spotsylvania_confirmed_case_transformed['new_cases'].apply(lambda x: x * 1.0 / Population_other)
df_spotsylvania_confirmed_case_transformed['daily_infection_rate_new_cases_moving_average_7_days'] = df_spotsylvania_confirmed_case_transformed['new_cases_moving_average_7_days'].apply(lambda x: x * 1.0 / Population_other)
df_spotsylvania_confirmed_case_transformed['daily_infection_rate_diff'] =df_spotsylvania_confirmed_case_transformed['daily_infection_rate'].diff()
df_spotsylvania_confirmed_case_transformed['daily_infection_rate_new_cases_moving_average_7_days_diff'] =df_spotsylvania_confirmed_case_transformed['daily_infection_rate_new_cases_moving_average_7_days'].diff()
df_spotsylvania_confirmed_case_transformed
df_palm_beach_confirmed_case_transformed.columns

joine_df=df_palm_beach_confirmed_case_transformed.merge(df_spotsylvania_confirmed_case_transformed, left_on='date', right_on='date',
          suffixes=('_palm_beach', '_spotsylvania'))

joine_df.columns
#

    
    
fig, ax = plt.subplots(figsize=(15,10))
ax.plot(joine_df['date'], joine_df['daily_infection_rate_new_cases_moving_average_7_days_diff_palm_beach'], color='C0')
ax.plot(joine_df['date'], joine_df['daily_infection_rate_new_cases_moving_average_7_days_diff_spotsylvania'], color='C1')


line0 = lines.Line2D([0], [0], label='Change in 7 days Moving average infection rate Palm Beach,FL', color='C0')
line1 = lines.Line2D([0], [0], label='Change in 7 days Moving average infection rate Spotsylvania, TX', color='C1')
span = ax.axvspan(date2num(datetime(2020,5,29)), date2num(datetime(2021,5,14)),color="C2", label = 'mask mandate in Spotsylvania Country')


plt.legend(handles=[
    line0,
    line1,
span])

ax.set_title("Change in 7 days moving average infection rate diff, Palm Beach, FL vs Spotsylvania, VA")
ax.set_xlabel("Date")
ax.set_ylabel("Infection Rate Diff")

plt.show()

#

    
    
fig, ax = plt.subplots(figsize=(15,10))
ax.plot(joine_df['date'], joine_df['daily_infection_rate_new_cases_moving_average_7_days_palm_beach'], color='C0')
ax.plot(joine_df['date'], joine_df['daily_infection_rate_new_cases_moving_average_7_days_spotsylvania'], color='C1')


line0 = lines.Line2D([0], [0], label='7 days Moving average infection rate Palm Beach', color='C0')
line1 = lines.Line2D([0], [0], label='7 days Moving average infection rate Spotsylvania', color='C1')
span = ax.axvspan(date2num(datetime(2020,5,29)), date2num(datetime(2021,5,14)),color="C2", label = 'mask mandate in Spotsylvania Country')


plt.legend(handles=[
    line0,
    line1,
span])

ax.set_title("7 days moving average infection rate, Palm Beach, FL vs Spotsylvania, VA")
ax.set_xlabel("Date")
ax.set_ylabel("Infection Rate")

plt.show()

# LOAD vaccinine data
df_vaccine = pd.read_csv('COVID-19_Vaccinations_in_the_United_States_County.csv')
df_vaccine

df_vaccine_palm_beach = df_vaccine[df_vaccine['FIPS'] == '12099'].reset_index(drop =True)
df_vaccine_palm_beach['Date'] = pd.to_datetime(df_vaccine_palm_beach['Date'])
df_vaccine_palm_beach
df_vaccine_palm_beach=df_vaccine_palm_beach.fillna(0)
df_vaccine_palm_beach
df_join_vaccine_data =df_palm_beach_confirmed_case_transformed.merge(df_vaccine_palm_beach, how="left", left_on='date', right_on='Date',
          suffixes=('', '_vaccine'))

df_join_vaccine_data=df_join_vaccine_data.fillna(0)
df_join_vaccine_data
df_join_vaccine_data.columns
#

    
    
fig, ax = plt.subplots(figsize=(15,10))
ax.plot(df_join_vaccine_data['date'], df_join_vaccine_data['daily_infection_rate_new_cases_moving_average_7_days'], color='C0')
ax.set_title("7 days moving average infection rate and Vaccine rate at Palm Beach, FL")
ax.set_xlabel("Date")
ax.set_ylabel("Infection Rate", color = 'C0')
ax.tick_params(axis='y', labelcolor='C0')

ax2 = ax.twinx()  # instantiate a second axes that shares the same x-axis
ax2.plot(df_join_vaccine_data['date'], df_join_vaccine_data['Series_Complete_Pop_Pct'], color='C1')
ax2.set_ylabel("Vaccine Rate", color = 'C1')
ax2.tick_params(axis='y', labelcolor='C1')

line0 = lines.Line2D([0], [0], label='7 days Moving average infection rate Palm Beach', color='C0')
line1 = lines.Line2D([0], [0], label='Vaccine rate Palm Beach', color='C1')

plt.legend(handles=[
    line0,
    line1])


plt.show()

# Jul 2021: DELTA started https://www.newsweek.com/first-us-covid-delta-variant-cases-how-did-it-mutate-1617871
# second reason: Summer break


# check correlation before vacacine available

before_vacacine = df_join_vaccine_data.loc[(df_join_vaccine_data['date'] > '2020-1-21') & (df_join_vaccine_data['date'] < '2020-12-31')]
filterd_df_before_vacacine = before_vacacine[['daily_infection_rate_new_cases_moving_average_7_days','Series_Complete_Pop_Pct']]
filterd_df_before_vacacine

correlations_before_vacacine = filterd_df_before_vacacine.corr()
correlations_before_vacacine
import seaborn as sns
sns.heatmap(correlations_before_vacacine)
plt.show()

# as expected, there is no correclation
# check correlation after vacacine available and before DELTA spike

after_vacacine = df_join_vaccine_data.loc[(df_join_vaccine_data['date'] > '2021-1-1') & (df_join_vaccine_data['date'] < '2021-6-30')]
filterd_df_after_vacacine = after_vacacine[['daily_infection_rate_new_cases_moving_average_7_days','Series_Complete_Pop_Pct']]
filterd_df_after_vacacine

correlations_after_vacacine = filterd_df_after_vacacine.corr()
correlations_after_vacacine


sns.heatmap(correlations_after_vacacine)
plt.show()

# very strong negative correlation
# check correlation after DELTA spike

after_spike = df_join_vaccine_data.loc[(df_join_vaccine_data['date'] > '2021-7-1') & (df_join_vaccine_data['date'] < '2022-1-1')]
filterd_df_after_spike = after_spike[['daily_infection_rate_new_cases_moving_average_7_days','Series_Complete_Pop_Pct']]
filterd_df_after_spike

correlations_after_spike = filterd_df_after_spike.corr()
correlations_after_spike

sns.heatmap(correlations_after_spike)
plt.show()

# slight negative correlation
after_vac = df_join_vaccine_data.loc[(df_join_vaccine_data['date'] > '2021-1-1')]

infection_rate = after_vac['daily_infection_rate_new_cases_moving_average_7_days']
infection_rate = infection_rate.to_numpy()
infection_rate
vaccine_rate = after_vac['Series_Complete_Pop_Pct']
vaccine_rate = vaccine_rate.to_numpy().reshape(-1, 1)
vaccine_rate
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import LinearRegression

from sklearn.tree import DecisionTreeRegressor
import numpy as np

model = DecisionTreeRegressor(max_depth=4).fit(vaccine_rate, infection_rate)

model.predict(np.array(90).reshape(-1, 1))
# daily new covid cases=
df_us_infection_rate=df_us_confirmed_case_transformed_with_pop.copy()
df_us_infection_rate['new_cases'] = df_us_confirmed_case_transformed_with_pop['cases'].diff()
df_us_infection_rate['new_cases_moving_average_7_days'] = df_us_infection_rate['new_cases'].rolling(window=7).mean().round()
df_us_infection_rate['daily_infection_rate'] = df_us_infection_rate['new_cases'] * 1.0 / df_us_confirmed_case_transformed_with_pop['Population_2020_Census']
df_us_infection_rate['daily_infection_rate_new_cases_moving_average_7_days'] = df_us_infection_rate['new_cases_moving_average_7_days']* 1.0 / df_us_confirmed_case_transformed_with_pop['Population_2020_Census']
df_us_infection_rate = df_us_infection_rate.fillna(0)
df_us_infection_rate
df_vaccine_clean = df_vaccine[['Date', 'Recip_County', 'Series_Complete_Pop_Pct']]
df_vaccine_clean['Date'] = pd.to_datetime(df_vaccine_clean['Date'])
df_vaccine_clean
df_us_infection = df_us_infection_rate.copy()
df_us_infection['County'] = df_us_infection['County'].apply(lambda x: "{} County".format(x))
df_us_infection
df_us_data  = df_us_infection.merge(df_vaccine_clean, how="left", left_on=['date', 'County'], right_on=['Date', 'Recip_County'],
          suffixes=('', '_vaccine'))
df_us_data.fillna(0)
df_us_data

us_after_vac = df_us_data.loc[(df_us_data['date'] > '2021-1-1')]
us_after_vac.fillna(0)

indices_to_keep = ~us_after_vac.isin([np.nan, np.inf, -np.inf]).any(1)
us_after_vac_cleaned = us_after_vac[indices_to_keep]
us_infection_rate = us_after_vac_cleaned['daily_infection_rate_new_cases_moving_average_7_days']
us_infection_rate = us_infection_rate.to_numpy()
us_vaccine_rate = us_after_vac_cleaned['Series_Complete_Pop_Pct']
us_vaccine_rate = us_vaccine_rate.to_numpy().reshape(-1, 1)


model = DecisionTreeRegressor(max_depth=4).fit(us_vaccine_rate, us_infection_rate)

model.predict(np.array(40).reshape(-1, 1))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11456875804555886, 'fraction_numerical': 0.02138019813063189, 'mean_word_length': 10.550096961861668, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 30, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '517008', 'license': 'MIT', 'n_tokens_mistral': 7136, 'n_tokens_neox': 6491, 'n_words': 907, 'path': 'A6-final project.ipynb', 'repo_name': 'IvyLinMS/data-512-a6', 'size': '17867'}