<jupyter_start><jupyter_text>Final Project CS209b EDA Flu data: <NAME>, <NAME>, <NAME> & <NAME><jupyter_code>import matplotlib.pyplot as plt
import numpy as np
import os
import sys
import pandas as pd
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
import csv
plt.rcParams.update({'font.size': 18})<jupyter_output><empty_output><jupyter_text>Data LoadingFirst we will load the data from all states into one single dataframe. wili stands for weekly influenza-like illness. I think the value corresponds to the percent of visits to healthcare providers that are for the flu (https://www.cdc.gov/flu/weekly/index.htm) but I'm not sure.<jupyter_code>flu_dir = '../flu_ground_truth/'
filename = 'wILI_cleaned_Alabama.csv'
with open(flu_dir + filename, 'r') as f:
    weeks = [row.split(',')[0] for row in f][1:]
week2cnt = {weeks[i]: i for i in range(len(weeks))}
location2info = {}
for filename in os.listdir(flu_dir):
    location = filename.replace('.csv', '').split('_')[-1]
    with open(flu_dir + filename, 'r') as f:
        rows = list(csv.reader(f))
        time2wili = {row[0]: float(row[1]) for row in rows[1:]}
        location2info[location] = time2wili
df = pd.DataFrame.from_dict(location2info)
df.rename(index=week2cnt, inplace=True)
df.sort_index(inplace=True)
df.head()
df.isna().sum()
df = df.fillna(df.mean())
df.isna().sum()
plt.figure(figsize=(20, 10))
for col in df.columns:
    data = df.loc[df[col].notnull(), col]
    plt.plot(list(data.index), list(data))
plt.show()
weeks = list(week2cnt.keys())
beginning_december_week = weeks.index('2010-48')
end_march_week = beginning_december_week + 17
month_avg = []
for idx in df.index:
    month_data = df.loc[idx, :]
    month_data = [x for x in month_data if not pd.isnull(x)]
    month_avg.append(np.mean(month_data))

plt.figure(figsize=(20, 10))
plt.plot(df.index, month_avg)
plt.xlabel('weeks since 2010-40')
plt.ylabel('')
plt.show()
correlation_df = df.corr()
correlation_df
mean_wili_df = pd.DataFrame(df.mean(), columns=['Mean WILI']).reset_index()
mean_wili_df = mean_wili_df.rename(columns={'index':'State_full'})
mean_wili_df
states_lat_long = pd.read_csv('../data/raw/statelatlong.csv')
states_lat_long = states_lat_long.rename(columns={"Latitude": "latitude", "Longitude": "longitude", "City":"State_full"})
states_lat_long
mean_wili_by_state_df = pd.merge(states_lat_long, mean_wili_df, how='inner', on='State_full')
correlation_df.loc['Texas']
corr_state_df = pd.DataFrame(correlation_df.loc['Texas']).reset_index().rename(columns={'index':'State_full'})
pd.merge(states_lat_long, corr_state_df, how='inner', on='State_full')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15724447772369898, 'fraction_numerical': 0.015350056158742045, 'mean_word_length': 5.994764397905759, 'pattern_counts': {'":': 3, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '553236', 'license': 'MIT', 'n_tokens_mistral': 1027, 'n_tokens_neox': 979, 'n_words': 208, 'path': 'notebooks/eda.ipynb', 'repo_name': 'benlevyx/modelling-infectious-disease', 'size': '2705'}