<jupyter_start><jupyter_text><jupyter_code>import pandas as pd

df = pd.read_csv('all-states-history.csv')
df.head()
import numpy as np
import datetime
from sklearn.preprocessing import OrdinalEncoder

def wrangle(df):
    temp = df.copy()
    
    temp['state'] = OrdinalEncoder().fit_transform(np.array(temp['state']).reshape(-1,1))
    
    quality_dict = {'F': 0, 'D': 1, 'C': 2, 'B': 3, 'A': 4, 'A+': 5}
    temp['dataQualityGrade'] = list(map(quality_dict.get, temp['dataQualityGrade']))
    
    temp2 = []
    for _, row in temp[['death', 'deathConfirmed', 'deathProbable']].iterrows():
        nan_list = list(map(np.isnan, row))
        if sum(nan_list) == 1:
            if nan_list[0]:
                row['death'] = row['deathConfirmed'] + row['deathProbable']
            elif nan_list[1]:
                row['deathConfirmed'] = row['death'] - row['deathProbable']
            else:
                row['deathProbable'] = row['death'] - row['deathConfirmed']
        temp2.append(row)
    temp[['death', 'deathConfirmed', 'deathProbable']] = temp2
    
    temp.interpolate(method = 'linear', limit_direction = 'backward', limit = 30, inplace = True)
    temp.fillna(0, inplace = True)
    
    temp['date'] = pd.to_datetime(temp['date'], format = '%Y%m%d')
    temp['date'] = temp['date'].map(datetime.datetime.toordinal)
    
    num_days = 10
    target = []
    for index, row in temp.iterrows():
        date = row['date']
        state = row['state']
        min_date = min(temp[temp['state'] == state]['date'])
        
        new_row = []
        for i in range(min(num_days, int(date - 4 - min_date))):
            previous_day_deaths = temp[(temp['date'] == date - i - 5) & (temp['state'] == state)]['positive'].values[0]
            new_row.append(previous_day_deaths)
        if len(new_row) < num_days:
            for i in range(num_days - len(new_row)):
                new_row.append(0)
        target.append(new_row)
    column_names = [str(i + 5) + ' Days Ago' for i in range(num_days)]
    target = pd.DataFrame(target, columns = column_names)
    
    temp.drop('date', axis = 1, inplace = True)
    
    return temp, target

X, y = wrangle(df)
X.head()
y.head()
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.25)
from sklearn.ensemble import RandomForestRegressor

model = RandomForestRegressor(n_estimators = 300,
                              max_features = 'sqrt',
                              max_depth = 14)
model.fit(X_train, y_train)
from sklearn.metrics import explained_variance_score as evs

y_pred = model.predict(X_train)
print("% of training variance explained: " + str(evs(y_train, y_pred)))

y_pred = model.predict(X_val)
print("% of validation variance explained: " + str(evs(y_val, y_pred)))
y_pred = model.predict(X_test)
print("% of test variance explained: " + str(evs(y_test, y_pred)))
y_pred = []
for _, row in y_test.iterrows():
    average = row.mean()
    new_row = [average for _ in range(len(row))]
    y_pred.append(new_row)
y_pred = pd.DataFrame(y_pred)

print("Baseline explained variance: " + str(evs(y_test, y_pred)))
print(model.predict(X_test))
print(pd.DataFrame(model.predict(X_test)).columns)
import matplotlib.pyplot as plt

def custom_pdp(df, model, target_var, gradations, title):
    if target_var not in df.columns:
        return None
    
    minimum = min(df[target_var])
    maximum = max(df[target_var])
    
    values = [minimum + i * (maximum - minimum) / (gradations - 1) for i in range(gradations)]
    
    predictions = []
    for value in values:
        df2 = df.copy()
        df2[target_var] = value
        y_pred = pd.DataFrame(model.predict(df2))
        averages = [y_pred[column].mean() for column in y_pred.columns]
        predictions.append(averages)
    
    plt.figure(figsize=(10, 6))
    plt.title(title)
    plt.plot(values, predictions)

custom_pdp(X_test, model, 'death', 10, 'Partial Dependence Plot for Death Count')
custom_pdp(X_test, model, 'hospitalized', 10, 'Partial Dependence Plot for Hospitalization Rate')
custom_pdp(X_test, model, 'positive', 10, 'Partial Dependence Plot for Patients Testing Positive')
custom_pdp(X_test, model, 'negative', 10, 'Partial Dependence Plot for Patients Testing Negative')
custom_pdp(X_test, model, 'totalTestResults', 10, 'Partial Dependence Plot for Patients Tested')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14149689991142603, 'fraction_numerical': 0.011514614703277236, 'mean_word_length': 3.3684719535783367, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '249073', 'license': 'MIT', 'n_tokens_mistral': 1603, 'n_tokens_neox': 1516, 'n_words': 350, 'path': 'Unit_2_Build_Week_Project.ipynb', 'repo_name': 'davidxu90/DS-Unit-2-Applied-Modeling', 'size': '4516'}