<jupyter_start><jupyter_text>VAR<jupyter_code>nobs = 1
df_train, df_test = df[0:-nobs], df[-nobs:]

# Check size
print(df_train.shape)  
print(df_test.shape) 
df_test 
# fit the model
model = VAR(df_train)
model_fit = model.fit() 
# Input data for forecasting
forecast_input = df_train.values[-nobs:]
forecast_input 
df_train.tail()
forecast_input == df_test.values
# Forecast
fc = model_fit.forecast(y=forecast_input, steps=nobs)
df_forecast = pd.DataFrame(fc, index=df.index[-nobs:], columns=df.columns)
df_forecast 
result = dict()
# check rmse 
for i in df.columns: 
    result[f'{i} RMSE'] = np.sqrt(mean_squared_error(df_forecast[i], df_test[i]))
    # print('rmse value for', i, 'is : ', np.sqrt(mean_squared_error(df_forecast[i], df_test[i])))
print(result) 
# make final predictions
model = VAR(df)
model_fit = model.fit() 
yhat = model_fit.forecast(model_fit.y, steps=150)
print(yhat.shape)  
len(yhat)
def forecast_to_df(data, forecast):
    ''' Convert forecast output array to dataframe using input data columns
    and last index '''
    date = data[-1:].index 
    future_date = []
    for i in range(len(forecast)): 
        date += timedelta(hours=1)
        future_date.append(date)
    yhat_df = pd.DataFrame(yhat, columns=data.columns, index=list(map(list, zip(*future_date))))
    # convert index values to datetime 
    idx = yhat_df.index.get_level_values(0).astype(str) 
    yhat_df.index = pd.to_datetime(idx)
    yhat_df.index.rename('Time', inplace=True)
    
    return yhat_df 
yhat_df = forecast_to_df(df, yhat)
yhat_df.head()
forecast_df = yhat_df.copy()
forecast_df.reset_index(level=0, inplace=True)
# forecast_df.style.hide_index() 
yhat_df.shape 
def make_plot(df1, df2, names):
    fig = make_subplots(rows=6, cols=1, subplot_titles=df1.columns,
                        vertical_spacing=0.05)

    for i, col in enumerate(df.columns):
        fig.add_trace(go.Scatter(name=names[0], x=df1.index, y=df1[col],  
                                marker = dict(size = 10, 
                                            color = 'blue'),
                                textfont=dict(
                                    color='black',
                                    size=18,  
                                    family='Times New Roman')),
                    row=i+1, col=1)
        fig.add_trace(go.Scatter(name=names[1], x=df2.index, y=df2[col],  
                                marker = dict(size = 10, 
                                            color = 'red')),
                    row=i+1, col=1) 
        
    fig.update_xaxes(
        rangeselector=dict(
            buttons=list([
                dict(count=1, label="1m", step="month", stepmode="backward"),
                dict(count=6, label="6m", step="month", stepmode="backward"),
                dict(count=1, label="YTD", step="year", stepmode="todate"),
                dict(count=1, label="1y", step="year", stepmode="backward"),
                dict(step="all")
            ])
        )
    )
    fig.update_layout(autosize=False,
        width=1300,
        height=1500,
        )
    fig.update_layout(legend=dict(
        orientation="h",
        yanchor="bottom",
        y=1.02,
        xanchor="right",
        x=1
    ))
    names = set()
    fig.for_each_trace(
        lambda trace:
            trace.update(showlegend=False)
            if (trace.name in names) else names.add(trace.name))

    fig.show() 
# make_plot(df, yhat_df, ['actual', 'forecast'])
# c = 1
# fig = plt.figure(figsize=(20,15))
# for col in df.columns:
#     plt.subplot(3, 2, c)
#     plt.title(f'{col}', size=15)
#     plt.tight_layout()
#     # plt.plot(df[8700:][col])
#     plt.plot(df_test[col], label='actual')
#     plt.plot(df_forecast[col], label='forecast')
#     plt.legend()
#     c += 1
# plt.show()<jupyter_output><empty_output><jupyter_text>LSTM<jupyter_code># https://machinelearningmastery.com/multivariate-time-series-forecasting-lstms-keras/
def series_to_supervised(data, n_in=1, n_out=1, dropnan=True):
	"""
	Frame a time series as a supervised learning dataset.
	Arguments:
		data: Sequence of observations as a list or NumPy array.
		n_in: Number of lag observations as input (X).
		n_out: Number of observations as output (y).
		dropnan: Boolean whether or not to drop rows with NaN values.
	Returns:
		Pandas DataFrame of series framed for supervised learning.
	"""
	n_vars = 1 if type(data) is list else data.shape[1]
	df = pd.DataFrame(data)
	cols, names = list(), list()
	# input sequence (t-n, ... t-1)
	for i in range(n_in, 0, -1):
		cols.append(df.shift(i))
		names += [('var%d(t-%d)' % (j+1, i)) for j in range(n_vars)]
	# forecast sequence (t, t+1, ... t+n)
	for i in range(0, n_out):
		cols.append(df.shift(-i))
		if i == 0:
			names += [('var%d(t)' % (j+1)) for j in range(n_vars)]
		else:
			names += [('var%d(t+%d)' % (j+1, i)) for j in range(n_vars)]
	# put it all together
	agg = pd.concat(cols, axis=1)
	agg.columns = names
	# drop rows with NaN values
	if dropnan:
		agg.dropna(inplace=True)
	return agg 
# normalize features
scaler = MinMaxScaler(feature_range=(0, 1))
scaled = scaler.fit_transform(df.values)
df_reframed = series_to_supervised(scaled) 
df_reframed 
df_reframed.columns 
# split into train and test sets
values = df_reframed.values
n_train_hours = 300 * 24 # first 300 days for train 
train = values[:n_train_hours, :]
test = values[n_train_hours:, :]
# split into input and outputs
train_X, train_y = train[:, :-6], train[:, -6:]
test_X, test_y = test[:, :-6], test[:, -6:] 
# reshape input to be 3D [samples, timesteps, features]
train_X = train_X.reshape((train_X.shape[0], 1, train_X.shape[1]))
test_X = test_X.reshape((test_X.shape[0], 1, test_X.shape[1]))
print(train_X.shape, train_y.shape, test_X.shape, test_y.shape) 
print(df_reframed.shape[0])
print(train_X.shape[0] + test_X.shape[0]) 
# make_plot(df.iloc[:n_train_hours, :], df.iloc[n_train_hours:, :], 'Training set', 'Test set')
# design network 
lstm_model = tf.keras.models.Sequential([
tf.keras.layers.LSTM(200, return_sequences=True, 
                            input_shape=(train_X.shape[1], train_X.shape[2])),
    tf.keras.layers.Dense(20, activation='tanh'),
    tf.keras.layers.LSTM(150),
    tf.keras.layers.Dense(20, activation='tanh'),
    tf.keras.layers.Dense(20, activation='tanh'),
    tf.keras.layers.Dropout(0.25),
    tf.keras.layers.Dense(6)
])
lstm_model.compile(optimizer='adam', loss='mae')
lstm_model.summary() 
model_path = 'models/LSTM.h5'
early_stopings = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=1, mode='min')
checkpoint = tf.keras.callbacks.ModelCheckpoint(model_path, monitor='val_loss', save_best_only=True, mode='min', verbose=0)
callbacks = [early_stopings,checkpoint] 
# fit network
history = lstm_model.fit(train_X, train_y, epochs=20, batch_size=72, 
                         validation_data=(test_X, test_y), verbose=2, 
                         shuffle=False, callbacks=callbacks) 
# plot history
plt.plot(history.history['loss'], label='train')
plt.plot(history.history['val_loss'], label='test')
plt.legend()
plt.show() 
# make a prediction
yhat = lstm_model.predict(test_X)
# invert scaling for forecast and actual 
inv_yhat = scaler.inverse_transform(yhat)
inv_y = scaler.inverse_transform(test_y)
# calculate RMSE
result_lstm = dict()
for i, col in enumerate(df.columns):
    result_lstm[f'{col} RMSE'] = np.sqrt(mean_squared_error(inv_y[:, i], inv_yhat[:, i])).round(3)
print(result_lstm) 
# c = 1
# fig = plt.figure(figsize=(20,15))
# for i, col in enumerate(df.columns):
#     plt.subplot(3, 2, c)
#     plt.title(f'{col}', size=15)
#     plt.tight_layout()
#     # plt.plot(df[8700:][col])
#     plt.plot(inv_yhat[:, i], label='forecast')
#     plt.plot(inv_y[:, i], label='actual')
#     plt.legend()
#     c += 1
# plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15179252479023647, 'fraction_numerical': 0.018815153826595474, 'mean_word_length': 3.2047033671833245, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '613481', 'license': 'BSD-3-Clause-LBNL', 'n_tokens_mistral': 3005, 'n_tokens_neox': 2854, 'n_words': 565, 'path': 'ml-pipeline/notebooks/train.ipynb', 'repo_name': 'esnet/netgraf', 'size': '7866'}