<jupyter_start><jupyter_text>Which COVID-19 Vaccine is Best for You? DS4A Team 55 Preliminary EDA<jupyter_code>import pandas as pd
import numpy as np
import math
import base64
import matplotlib.pyplot as plt
import seaborn as sns
from sodapy import Socrata
# increase the size and quality of any plots we produce
import matplotlib as mpl
mpl.rcParams["figure.dpi"] = 300<jupyter_output><empty_output><jupyter_text>state_vax.csvVaccine administration data for states within the U.S., including the District of Columbia from 12/13/2020 to the present, updated daily.<jupyter_code>client = Socrata(
        domain = "data.cdc.gov",
        app_token = "bsY<PASSWORD>KAQhIIVd7wzBzp<PASSWORD>",
        timeout=10
        )
results = client.get_all("unsk-b7fc")

# convert to pandas DataFrame
df = pd.DataFrame.from_records(results)
df.sample(10)
# df.info()
# remove all distribution cols
# remove "Administered", this is the total for 1st dose and complete series
# remove all 5plus cols, very recently approved and sporadic across states
# remove additional doses for 50plus, age group doesn't show up elsewhere in dataset


df = df.drop(columns= ["mmwr_week","distributed", "distributed_janssen", "distributed_moderna", 
                       "distributed_pfizer", "distributed_unk_manuf", "dist_per_100k","distributed_per_100k_12plus",
                       "distributed_per_100k_18plus", "distributed_per_100k_65plus", "distributed_per_100k_5plus",
                       "administered","administered_dose1_recip_5plus",
                       "administered_dose1_recip_5pluspop_pct", "series_complete_5plus", "series_complete_5pluspop_pct", 
                       "administered_5plus", "admin_per_100k_5plus", "series_complete_moderna_5plus",
                       "series_complete_pfizer_5plus", "series_complete_janssen_5plus", "series_complete_unk_manuf_5plus",
                       "additional_doses_50plus", "additional_doses_50plus_vax_pct"], axis=1)
# abbreviating column names for readability
# cols in api data are labelled differently than csv directly from data.cdc.gov, values match up so renaming those cols

rename_cols = {"administered_12plus":"admin_12plus", 
               "administered_18plus":"admin_18plus",
               "administered_65plus":"admin_65plus",
               "administered_janssen":"admin_janssen",
               "administered_moderna":"admin_moderna",
               "administered_pfizer":"admin_pfizer",
               "administered_unk_manuf":"admin_unk",
               "recip_administered":"recip_admin",
               "administered_dose1_recip":"admin_dose1",
               "administered_dose1_pop_pct":"admin_dose1_pop_pct",
               "administered_dose1_recip_1":"admin_dose1_12plus",
               "administered_dose1_recip_2":"admin_dose1_12plus_pop_pct",
               "administered_dose1_recip_3":"admin_dose1_18plus",
               "administered_dose1_recip_4":"admin_dose1_18plus_pop_pct",
               "administered_dose1_recip_5":"admin_dose1_65plus",
               "administered_dose1_recip_6":"admin_dose1_65plus_pop_pct",
               "series_complete_yes":"series_complete",
               "series_complete_12pluspop":"series_complete_12plus_pop_pct",
               "series_complete_18pluspop":"series_complete_18plus_pop_pct",
               "series_complete_65pluspop":"series_complete_65plus_pop_pct",
               "series_complete_unk_manuf":"series_complete_unk",
               "series_complete_unk_manuf_1":"series_complete_unk_12plus",
               "series_complete_unk_manuf_2":"series_complete_unk_18plus",
               "series_complete_unk_manuf_3":"series_complete_unk_65plus",
               "series_complete_12pluspop_pct":"series_complete_12plus_pop_pct",
               "series_complete_18pluspop_pct":"series_complete_18plus_pop_pct",
               "series_complete_65pluspop_pct":"series_complete_65plus_pop_pct"
               }
df = df.rename(columns=rename_cols)
# df.columns
# see data types of df
# all types = object, need to change to ints and floats
# df.dtypes
# show cols with NaNs
# cols with NaNs are additional dose cols, after investigating this happens when other additional dose cols are 0
df.columns[df.isnull().any()]
# replace NaNs with 0 to fill in values
nan_col = ['additional_doses', 'additional_doses_vax_pct',
           'additional_doses_18plus', 'additional_doses_18plus_vax_pct',
           'additional_doses_65plus', 'additional_doses_65plus_vax_pct',
           'additional_doses_moderna', 'additional_doses_pfizer',
           'additional_doses_janssen', 'additional_doses_unk_manuf']
df[nan_col] = df[nan_col].fillna(0)
to_int = ['admin_12plus', 'admin_18plus', 'admin_65plus','admin_janssen',
          'admin_moderna', 'admin_pfizer', 'admin_unk','admin_per_100k',
          'admin_per_100k_12plus', 'admin_per_100k_18plus','admin_per_100k_65plus',
          'recip_admin', 'admin_dose1', 'admin_dose1_12plus', 'admin_dose1_18plus',
          'admin_dose1_65plus', 'series_complete_12plus', 'series_complete_18plus',
          'series_complete','series_complete_65plus', 'series_complete_janssen', 'series_complete_moderna',
          'series_complete_pfizer','series_complete_unk', 'series_complete_janssen_12plus',
          'series_complete_moderna_12plus', 'series_complete_pfizer_12plus', 'series_complete_unk_12plus',
          'series_complete_janssen_18plus', 'series_complete_moderna_18plus', 'series_complete_pfizer_18plus',
          'series_complete_unk_18plus', 'series_complete_janssen_65plus', 'series_complete_moderna_65plus',
          'series_complete_pfizer_65plus', 'series_complete_unk_65plus', 'additional_doses',
          'additional_doses_18plus', 'additional_doses_65plus', 'additional_doses_moderna',
          'additional_doses_pfizer', 'additional_doses_janssen', 'additional_doses_unk_manuf'
         ]

to_float = ['admin_dose1_pop_pct', 'admin_dose1_12plus_pop_pct', 'admin_dose1_18plus_pop_pct',
            'admin_dose1_65plus_pop_pct', 'series_complete_pop_pct', 'series_complete_12plus_pop_pct',
            'series_complete_12plus_pop_pct', 'series_complete_18plus_pop_pct', 'series_complete_65plus_pop_pct',
            'additional_doses_vax_pct', 'additional_doses_18plus_vax_pct', 'additional_doses_65plus_vax_pct'
           ]

df[to_int] = df[to_int].astype("int")
df[to_float] = df[to_float].astype("float")
# checking that data types hace been converted correctly
# df.dtypes
# removing locations outside of the contential united states as well as united states total

drop_location = ["AS", "BP2", "DD2", "FM", "GU",
                 "IH2", "MH", "MP", "PR", "RP",
                 "US", "VA2", "VI", "LTC"]
df = df[~df["location"].isin(drop_location)]

# checking that location vals = 51, including DC

df.location.nunique()
# converting "date" col to datetime
df["date"] = pd.to_datetime(df["date"])
# filter location to New York
df_ny = df.loc[df["location"] == "NY"]
# set index to date to extract rows for the first of each month, aligns with how other datasets are reported
df_ny_m = df_ny.set_index("date")
df_ny_m = df_ny_m.groupby(df_ny_m.index.strftime("%Y-%m")).tail(1)
df_ny_m.head(10)
# first reported date and most recent date for vax data, updated daily
print(df_ny["date"].min())
print(df_ny["date"].max())
# trend in administration of each vax manufacturer 
ny_m_plt = df_ny_m.reset_index()
ny_m_plt.plot(x="date", y=["admin_janssen", "admin_moderna", "admin_pfizer", "admin_unk"], kind="line")<jupyter_output><empty_output><jupyter_text>In the "Rates of COVID-19 Vaccine Administration by Manufacturer" graph, above, Pfizer and Moderna are the most administered vaccines with Pfizer being the highest. Rates of J&J and vaccines by an unknown manufacturer are very low compared to Pfizer and Moderna. The J&J vaccine has a steady and slow upwards trend as the year progresses.---<jupyter_code>ny_m_plt.plot(x="date", y=["admin_12plus", "admin_18plus", "admin_65plus"], kind="line", title="Rate of COVID-19 Vaccine Administration by Age Group")<jupyter_output><empty_output><jupyter_text>In the "Rates of COVID-19 Vaccine Administration by Age Group" graph, above, age group 18plus is the most vaccinated until 12plus spikes in May of 2021 and passes 18plus shortly after. Age group 65plus remains the lowest with a sharp increase in March of 2021 and has been steadily increasing since then. However, this graph only tells part of the story. We should look into the percent of population per each age group to get a better understanding of vaccination trends.---<jupyter_code>ny_m_plt.plot(x="date", y=["admin_per_100k_12plus", "admin_per_100k_18plus", "admin_per_100k_65plus"], kind="line", title="Rates of COVID-19 Vaccine Administration by 100k per Age Group")<jupyter_output><empty_output><jupyter_text>Comparing the "Rates of COVID-19 Vaccine Administration by 100k per Age Group" graph to the graph showing only the value counts for each age group reveals new insights. In the per 100k graph, the 65plus age group remains the highest vaccinated group by a good margin throughout the reported timeframe. Once people 12plus were able to get vaccinated in May 2021 the rate of vaccination for this group rose almost immediately and lies just below the rate of the 18plus group.---<jupyter_code>ny_m_plt.plot.bar(x="date", y=["admin_12plus", "admin_18plus", "admin_65plus"])<jupyter_output><empty_output><jupyter_text>covid_casesCovid cases in NY<jupyter_code>#read data
covid_df = pd.read_csv("data/COVID-19_Case_Surveillance_Public_Use_Data_with_Geography.csv", low_memory = False)
covid_df.columns #column names
covid_df.dtypes
covid_df.shape
#data cleaning
covid_df['case_month'] = pd.to_datetime(covid_df['case_month']) #convert to date
covid_df.head()
covid_df.sample(10, random_state = 42)
covid_df.describe(include = 'all',datetime_is_numeric=True) #describe and include string columns
sns.pairplot(covid_df)
covid_df.boxplot('case_onset_interval')
#covid cases trend
trend_df = covid_df.groupby(['case_month']).size().reset_index(name = 'case_count').set_index('case_month')
trend_df['case_count_per_100k'] = trend_df['case_count']/100000
plt.plot(trend_df.index,trend_df['case_count_per_100k'])
plt.title('COVID-19 Cases Have Spiked Twice And Is Trending Upwards')
plt.xlabel('Date')
plt.ylabel('Case Count (Hundred Thousand)')
plt.xticks(rotation=90)
plt.show()
covid_df.groupby(['race']).size().plot.bar()
plt.title('Individuals who identify as White account for majority of cases')
plt.xlabel('Race')
plt.ylabel('Case Count (Hundred Thousand)')
plt.xticks(rotation=90)
plt.show()
#death status per race
death_df = covid_df[covid_df['death_yn'] == 'Yes'] 
death_df.groupby(['race']).size().plot.bar()
plt.title('Individuals who identify as White account for majority of the COVID deaths')
plt.xlabel('Race')
plt.ylabel('Number of COVID Deaths')
plt.xticks(rotation=90)
plt.show()
#death status per state
death_df.groupby(['res_county']).size().reset_index(name = 'count')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10218713131863146, 'fraction_numerical': 0.032398584263544784, 'mean_word_length': 4.979381443298969, 'pattern_counts': {'":': 27, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '470137', 'license': 'CC0-1.0', 'n_tokens_mistral': 3890, 'n_tokens_neox': 3528, 'n_words': 790, 'path': 'team55_prelim_eda.ipynb', 'repo_name': 'shinaolaatoro/ds4a-team55', 'size': '11007'}