<jupyter_start><jupyter_text>Read the data<jupyter_code>dataset = pd.read_csv("../../Data/all_stocks_5yr.csv")

# Changing the date column to the datetime format (best format to work with time series)
dataset['Date'] = [dt.datetime.strptime(d,'%Y-%m-%d').date() for d in dataset['Date']]

dataset.head()
# Assigning a mid price column with the mean of the Highest and Lowest values

dataset['Mid'] = (dataset['High'] + dataset['Low'])/2
dataset.head()
# Getting rid of null columns

missing_data = pd.DataFrame(dataset.isnull().sum()).T
print(missing_data)

for index, column in enumerate(missing_data.columns):
    if missing_data.loc[0][index] != 0:
        dataset = dataset.drop(dataset.loc[dataset[column].isnull()].index)
        
missing_data = pd.DataFrame(dataset.isnull().sum()).T
print(missing_data)
# Let's analyze 3M stocks a bit deeper

MMM_stocks = dataset[dataset['Name'] == 'MMM']
# Creating a percent change column related to the closing price

percent_change_closing_price = MMM_stocks['Close'].pct_change()
percent_change_closing_price.fillna(0, inplace=True)
MMM_stocks['PC_change'] = pd.DataFrame(percent_change_closing_price)

# As we want to predict the closing price, let's add the target column as the close price shifted by 1

MMM_stocks['Target'] = MMM_stocks['Close'].shift(-1)

MMM_stocks = MMM_stocks.drop(0, axis = 0)
MMM_stocks = MMM_stocks.drop('Name', axis = 1)
MMM_stocks = MMM_stocks.drop('Date', axis = 1)

MMM_stocks.head()<jupyter_output><empty_output><jupyter_text>We gonna first focus on univariate predictions<jupyter_code>series = MMM_stocks['Close'].values
window_size = 10
dcmpstn = seasonal_decompose(series, model='additive', freq=window_size)
dcmpstn.plot()
plt.show()
#plt.savefig('decomp.pdf')
plt.figure(figsize=(10,6))
plt.subplot(4,1,1)
plt.plot(dcmpstn.observed)
plt.ylabel('Observed')
plt.subplot(4,1,2)
plt.plot(dcmpstn.trend)
plt.ylabel('Trend')
plt.subplot(4,1,3)
plt.plot(dcmpstn.seasonal)
plt.ylabel('Seasonal')
plt.subplot(4,1,4)
plt.plot(dcmpstn.resid)
plt.ylabel('Residual')
plt.savefig('decomp.pdf')
plt.show()<jupyter_output><empty_output><jupyter_text>Now, let's try to model the trend and seasonal components separately<jupyter_code># It's worth mentioning that we had lost 'window_size' samples in both ends of the series
# due to the performed transformation
trend = dcmpstn.trend[window_size:-window_size]
seasonal = dcmpstn.seasonal[window_size:-window_size]
print('Length:', len(trend))

# Therefore, we are going to limit our analysis only to the remaining values
series_c = series[window_size:-window_size]<jupyter_output>Length: 1236<jupyter_text>The trend component will be predicted using a differentiated version of the series<jupyter_code># Uses the statsmodels diff method to get the first derivative
trend_diff = diff(trend)
print('Length:', len(trend_diff))
f, ax = plt.subplots()
ax.plot(trend_diff)
plt.show()
# Train and test split
split_point = 1000
train_trend_orig = trend_diff[:split_point]
test_trend_orig = trend_diff[split_point:]

train_seasonal_orig = seasonal[:split_point]
test_seasonal_orig = seasonal[split_point:]

# Getting data scalers
t_scaler = StandardScaler()
train_trend = t_scaler.fit_transform(np.array(train_trend_orig).reshape(-1, 1))[:,0]
test_trend = t_scaler.transform(np.array(test_trend_orig).reshape(-1, 1))[:,0]

s_scaler = StandardScaler()
train_seasonal = s_scaler.fit_transform(np.array(train_seasonal_orig).reshape(-1, 1))[:,0]
test_seasonal = s_scaler.transform(np.array(test_seasonal_orig).reshape(-1, 1))[:,0]
# Preparing data for the experiment with an univariate model 
# Getting Closing Price and arranging lists for training/testing based on the sequence

# Thanks Mr. Angelo for this very useful helper :D
def split_sequence(sequence, n_steps):
    X, y = list(), list()
    for i in range(len(sequence)):
        # find the end of this pattern
        end_ix = i + n_steps
        # check if we are beyond the sequence
        if end_ix > len(sequence)-1:
            break
        # gather input and output parts of the pattern
        seq_x, seq_y = sequence[i:end_ix], sequence[end_ix]
        X.append(seq_x)
        y.append(seq_y)  
    return np.array(X), np.array(y)
# Trend component
ttr_x, ttr_y = split_sequence(train_trend, window_size)
# For the test, we add the last samples from training as the head of the list to obtain
# the same size of testing samples
aux_tts = train_trend[-(window_size + 1):].copy().tolist()
# Above, I added an extra training sample (window_size + 1) to compensate the effect of the differentiated data, 
# which looses one sample
aux_tts.extend(test_trend)
tts_x, tts_y = split_sequence(aux_tts, window_size)

# Seasonal component
str_x, str_y = split_sequence(train_seasonal, window_size)
aux_sts = train_seasonal[-window_size:].copy().tolist()
aux_sts.extend(test_seasonal)
sts_x, sts_y = split_sequence(aux_sts, window_size)
tm = Sequential()
tm.add(LSTM(50, activation='relu', return_sequences=True, input_shape=(20, 1)))
tm.add(LSTM(50, activation='relu'))
tm.add(Dense(1))
tm.compile(optimizer='adam', loss='mse')


plot_model(tm, to_file='model.png', show_shapes=True, show_layer_names=True)
# Getting the predictions models
n_features = 1

# Trend model
# Define model
tm = Sequential()
tm.add(LSTM(50, activation='relu', return_sequences=True, input_shape=(window_size, n_features)))
tm.add(LSTM(50, activation='relu'))
tm.add(Dense(1))
tm.compile(optimizer='adam', loss='mse')

X_t = ttr_x.reshape(ttr_x.shape[0], ttr_x.shape[1], n_features)


# Fit trend model
_ = tm.fit(X_t, ttr_y, epochs=100, verbose=0)


# Seasonal model
# Define model
sm = Sequential()
sm.add(LSTM(50, activation='relu', return_sequences=True, input_shape=(window_size, n_features)))
sm.add(LSTM(50, activation='relu'))
sm.add(Dense(1))
sm.compile(optimizer='adam', loss='mse')

X_s = str_x.reshape(str_x.shape[0], str_x.shape[1], n_features)

# Fit trend model
_ = sm.fit(X_s, str_y, epochs=100, verbose=0)<jupyter_output>WARNING:tensorflow:From /home/mastelini/Documents/Data_Science_for_Finance/py_env/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
WARNING:tensorflow:From /home/mastelini/Documents/Data_Science_for_Finance/py_env/lib/python3.7/site-packages/tensorflow/python/keras/utils/losses_utils.py:170: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
WARNING:tensorflow:From /home/mastelini/Documents/Data_Science_for_Finance/py_env/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.<jupyter_text>Getting the predictions for each component<jupyter_code># Trend
trend_preds_diff = tm.predict(tts_x.reshape((len(tts_x), window_size, n_features)))[:, 0]
trend_preds_diff = t_scaler.inverse_transform(np.array(trend_preds_diff).reshape(-1, 1))[:,0]

# Now we apply the predictions of the differences (first derivative) over the lagged trend data
# Include the last point of training to enable getting predictions for the first point in testing
aux_tts_trend = [trend[split_point - 1].tolist()]
aux_tts_trend.extend(trend[split_point:-1].tolist())

# Sums the time lagged observations with the estimate of variation (first derivative)
trend_preds = []
for i in range(len(aux_tts_trend)):
    trend_preds.append(aux_tts_trend[i] + trend_preds_diff[i])
    

# Seasonal
seasonal_preds = sm.predict(sts_x.reshape((len(sts_x), window_size, n_features)))[:, 0]
seasonal_preds = s_scaler.inverse_transform(np.array(seasonal_preds).reshape(-1, 1))[:,0]
f, ax = plt.subplots()
ax.plot(trend[split_point:])
ax.plot(trend_preds)
ax.set_title('Trend prediction')
plt.show()

print('MSE:', mean_squared_error(trend[split_point:], trend_preds))
f, ax = plt.subplots()
ax.plot(seasonal[split_point:])
ax.plot(seasonal_preds)
ax.set_title('Seasonal prediction')
plt.show()

print('MSE:', mean_squared_error(seasonal[split_point:], seasonal_preds))<jupyter_output><empty_output><jupyter_text>Final predictions<jupyter_code>preds = [t + s for t, s in zip(trend_preds, seasonal_preds)]
f, ax = plt.subplots()
ax.plot(series_c[split_point:])
ax.plot(preds)
ax.set_title('Final prediction')
plt.show()

print('MSE:', mean_squared_error(series_c[split_point:], preds))
print('TU:', TU(series_c[split_point:], preds))
print('POCID:', POCID(series_c[split_point:], preds))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12886952533822518, 'fraction_numerical': 0.012726438890162806, 'mean_word_length': 6.44283276450512, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '325604', 'license': 'MIT', 'n_tokens_mistral': 3234, 'n_tokens_neox': 3104, 'n_words': 689, 'path': 'src/mtr/LSTM_and_decomposition-Univariate.ipynb', 'repo_name': 'angelomenezes/Data_Science_for_Finance', 'size': '8722'}