<jupyter_start><jupyter_text>Imports<jupyter_code>import numpy as np
import tensorflow as tf
from tensorflow import keras
import pandas as pd
import seaborn as sns
from pylab import rcParams
import matplotlib.pyplot as plt
from matplotlib import rc
import tensorflow
from tensorflow.keras import layers
import tensorflow as tf
import sklearn
from sklearn import preprocessing
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error
import math 
import pandas
import pandas as pd
from matplotlib import pyplot
import numpy as np
import matplotlib.pyplot as plt
import numpy
import seaborn as sns
from tqdm import tqdm<jupyter_output><empty_output><jupyter_text>Get the list of states<jupyter_code>dataset = pandas.read_csv('deaths_7dav_incidence_prop-2020-03-01-to-2021-03-05.csv')
    
states = set(dataset['geo_value'])
print(states)
statee = 'wy'
states.remove(statee)
states.remove('ak')<jupyter_output><empty_output><jupyter_text>Preparing training data<jupyter_code>from tqdm import tqdm 
def get_timeseries_inputs(data, window_size, horizon,
                   multivariate_output=False, shuffle=False, other_horizon=None):
    """
    :param data: numpy.array
        shape (n_samples, n_features) or (M, n_samples, n_features)
    :param window_size: int
        Fixed size of the look-back  
    :param horizon: int
        Forecasting horizon, the number of future steps that have to be forecasted
    :param multivariate_output: if True, the target array will not have shape
        (n_samples, output_sequence_len) but (n_samples, output_sequence_len, n_features)
        
    :param shuffle: if True shuffle the data on the first axis
    :param other_horizon:
    :return: tuple
        Return two numpy.arrays: the input and the target for the model.
        the inputs has shape (n_samples, input_sequence_len, n_features)
        the target has shape (n_samples, output_sequence_len)
    """
    if data.ndim == 2:
        data = np.expand_dims(data, 0)
    inputs = []
    targets = []
    for X in tqdm(data):  # for each array of shape (n_samples, n_features)
        n_used_samples = X.shape[0] - horizon - window_size + 1
        for i in range(n_used_samples):
            inputs.append(X[i: i + window_size])
            # TARGET FEATURE SHOULD BE THE FIRST
            if multivariate_output:
                if other_horizon is None:
                    targets.append(
                        X[i + window_size: i + window_size + horizon])
                else:
                    targets.append(
                        X[i + 1: i + window_size + 1])
            else:
                if other_horizon is None:
                    targets.append(
                        X[i + window_size: i + window_size + horizon, 0])
                else:
                    targets.append(
                        X[i + 1: i + window_size + 1, 0])
    encoder_input_data = np.asarray(inputs)  # (n_samples, sequence_len, n_features)
    decoder_target_data = np.asarray(targets)  # (n_samples, horizon) or (n_samples, horizon, n_features) if multivariate_output
    idxs = np.arange(encoder_input_data.shape[0])
    if shuffle:
        np.random.shuffle(idxs)
    return encoder_input_data[idxs], decoder_target_data[idxs]
def get_state(state):
    dataset = pandas.read_csv('deaths_7dav_incidence_prop-2020-03-01-to-2021-03-05.csv')
    
    value_dataset = dataset[dataset['geo_value']== state]
    value_dataset = value_dataset[['time_value','value']]
    value_dataset_copy = value_dataset.copy()
    value_dataset_copy['time_value']=pandas.to_datetime(value_dataset_copy['time_value'])
    value_dataset_copy.set_index('time_value',inplace = True)
    value_dataset_copy = value_dataset_copy.truncate(before='2020-4-1',after='2021-3-1')
    
    scaler = StandardScaler()
    scaler = scaler.fit(value_dataset_copy)
    data_scaled = scaler.transform(value_dataset_copy)
    return data_scaled
def get_state_tiem(state):
    data_scaled = get_state(state)
    window_size = 45
    horizon = 1
    X_series,Y_series = get_timeseries_inputs(data_scaled, window_size, horizon)
    
    return X_series,Y_series
a,b = get_state_tiem('ak')
def get_final(states):
    X_all= a
    #X_all.reshape(1,45,1)
    #Y_all= pandas.Series()
    Y_all =b
    for state in states:
        X_series,Y_series = get_state_tiem(state)
        X_all = np.concatenate((X_all,X_series),axis = 0)
        Y_all = np.concatenate((Y_all,Y_series),axis = 0)
    
    return X_all,Y_all
X_all,Y_all = get_final(states)
print(X_all.shape)
Y_all.shape<jupyter_output><empty_output><jupyter_text>Build and fit the model<jupyter_code>model = tf.keras.Sequential([
    layers.LSTM(32,activation = 'relu',input_shape= (X_all.shape[1],X_all.shape[2]),return_sequences = True),
    layers.LSTM(32,activation = 'relu',return_sequences = False),
    layers.Dropout(0.2),
    layers.Dense(Y_all.shape[1])
])

model.compile(
optimizer = tf.keras.optimizers.Adam(0.001),
loss = tf.keras.metrics.mse,
)
model.summary()
history = model.fit(X_all,Y_all,epochs=30,batch_size=64, validation_split=0.1,verbose=2,shuffle= True)
plt.figure(figsize=(14,7))
plt.plot(history.history['loss'],label= 'training loss')
plt.plot(history.history['val_loss'],label = 'validation loss')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Preparing test data of state wy<jupyter_code>def get_test(state):
    dataset = pandas.read_csv('deaths_7dav_incidence_prop-2020-03-01-to-2021-03-05.csv')
    
    value_dataset = dataset[dataset['geo_value']== state]
    value_dataset = value_dataset[['time_value','value']]
    value_dataset_copy = value_dataset.copy()
    value_dataset_copy['time_value']=pandas.to_datetime(value_dataset_copy['time_value'])
    value_dataset_copy.set_index('time_value',inplace = True)
    value_dataset_copy = value_dataset_copy.truncate(before='2020-4-1',after='2021-3-1')
    
    return value_dataset_copy
dd_test = get_test(statee)
dd_test
scaler_test = StandardScaler()
scaler_test = scaler_test.fit(dd_test)
data_scaled_test = scaler_test.transform(dd_test)
window_size = 45
horizon = 1
X_series_test,Y_series_test = get_timeseries_inputs(data_scaled_test, window_size, horizon)
print(X_series_test.shape)
print(Y_series_test.shape)<jupyter_output>(290, 45, 1)
(290, 1)<jupyter_text>Forcast<jupyter_code>forcast = model.predict(X_series_test)
print(forcast.shape)
print(forcast)
y_pred_true = scaler_test.inverse_transform(forcast)
y_pred_true
len(y_pred_true)
dd_test_true = dd_test[45:]
dd_test_true.index
rmse = math.sqrt(mean_squared_error(dd_test_true['value'],y_pred_true))
print('Test RMSE : %.3f ' % rmse)
rmsee = str(round(rmse,3))
forecast_period_dates = pd.date_range(('2020-05-16 00:00:00'), periods=290, freq='1d').tolist()
forecast_period_dates
forecast_dates = []
for time_i in forecast_period_dates:
    forecast_dates.append(time_i.date())
forecast_dates
y_pred_true_list = list(y_pred_true)
    
df_forecast_x = pd.DataFrame({'Date':np.array(forecast_dates), 'value':y_pred_true_list})<jupyter_output><empty_output><jupyter_text>Graph<jupyter_code>plt.figure(figsize=(14,7))
plt.title('Prediction of mortality(deaths per 100,000 people) in the state ' + statee+ ' with rmse '+ rmsee )
plt.plot(df_forecast_x['Date'], df_forecast_x['value'],label="predict")
#plt.plot(test['time_value'], test['value'],"_-",label="ture")
plt.plot(dd_test_true['value'],label="original")
"""open the grid"""
plt.grid(True)
plt.legend()
#plt.legend(bbox_to_anchor=(1.0, 1), loc=1, borderaxespad=0.)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11170770036668413, 'fraction_numerical': 0.021870089051859613, 'mean_word_length': 4.329378925331472, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '206915', 'license': 'MIT', 'n_tokens_mistral': 2811, 'n_tokens_neox': 2633, 'n_words': 511, 'path': 'Part2-Mobility-analysis/3.2 Univariarate Prediction at State Level (wy).ipynb', 'repo_name': 'tonywu71/Excess-Mortality-Covid', 'size': '7636'}