<jupyter_start><jupyter_text>Prepare dataset<jupyter_code>assets = ['MAXR.TO']
df = yf.download(assets, period="max")
df
# Create SMA and EMA indicators
def SMA (table, period=30, col='Adj Close'):
    return table[col].rolling(window=period).mean()

def EMA(table, period=24, col='Adj Close'):
    return table[col].ewm(span=period, adjust=False).mean()
# Create a functin to calculate MACD
def MACD(table, period_l=26, period_s=12, period_signal=9, col='Adj Close'):
    ShortEMA = EMA(table, period=period_s, col=col) # Short term EMA
    LongEMA = EMA(table, period=period_l, col=col) # long term EMA
    
    # Calculate and store the MACD into the df
    table['MACD'] = ShortEMA - LongEMA
    
    # Calculate and store the signal line into the df
    table['signal_Line'] = EMA(table, period=period_signal, col='MACD')
    
    return table
# Create a function to calculate the Relative Strength Index (RSI)
def RSI(table, period=14, col='Adj Close'):
    delta = table[col].diff(1)
    delta = delta.dropna()
    up = delta.copy()
    down = delta.copy()
    up[up < 0] = 0
    down[down > 0] = 0
    table['up'] = up
    table['down'] = down
    Avg_Gain = SMA(table, period, col='up')
    Avg_Loss = abs(SMA(table, period, col='down'))
    RS = Avg_Gain / Avg_Loss
    RSI = 100.0 - (100.0 / (1.0+RS))
    table['RSI'] = RSI
    
    return table
# Add the indicators to the data set
MACD(df)
RSI(df)
df['SMA'] = SMA(df)
df['EMA'] = EMA(df)
# Create a target col
df['Target'] = np.where(df['Adj Close'].shift(-1) > df['Adj Close'], 1, 0)
# Remove Nulls
df = df.dropna()
df
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
# Build Xs and Y
col_kept = ['Adj Close', 'MACD', 'signal_Line', 'RSI', 'SMA', 'EMA', 'Volume']
X = df[col_kept].values
Y = df['Target'].values
# Split the train/test
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, shuffle=False)<jupyter_output><empty_output><jupyter_text>First, try Decision Tree<jupyter_code># Decision Tree classifier
tree = DecisionTreeClassifier(random_state=0, 
                              min_impurity_decrease = 0.00001,
                              criterion='entropy').fit(X_train, Y_train)
path = tree.cost_complexity_pruning_path(X_train, Y_train)
ccp_alphas, impurities = path.ccp_alphas, path.impurities<jupyter_output><empty_output><jupyter_text>Cost complexity pruning is to control the size of a tree. In DecisionTreeClassifier, this pruning technique is parameterized by the cost complexity parameter, ccp_alpha. Greater values of ccp_alpha increase the number of nodes pruned. Here we only show the effect of ccp_alpha on regularizing the trees and how to choose a ccp_alpha based on validation scores.<jupyter_code>fig, ax = plt.subplots()
ax.plot(ccp_alphas[:-1], impurities[:-1], marker='o', drawstyle="steps-post")
ax.set_xlabel("effective alpha")
ax.set_ylabel("total impurity of leaves")
ax.set_title("Total Impurity vs effective alpha for training set")
clfs = []
for ccp_alpha in ccp_alphas:
    clf = DecisionTreeClassifier(random_state=0, 
                                 min_impurity_decrease = 0.00001,
                                 criterion='entropy',
                                 ccp_alpha=ccp_alpha)
    clf.fit(X_train, Y_train)
    clfs.append(clf)
print("Number of nodes in the last tree is: {} with ccp_alpha: {}".format(
      clfs[-1].tree_.node_count, ccp_alphas[-1]))
clfs = clfs[:-1]
ccp_alphas = ccp_alphas[:-1]

node_counts = [clf.tree_.node_count for clf in clfs]
depth = [clf.tree_.max_depth for clf in clfs]
fig, ax = plt.subplots(2, 1)
ax[0].plot(ccp_alphas, node_counts, marker='o', drawstyle="steps-post")
ax[0].set_xlabel("alpha")
ax[0].set_ylabel("number of nodes")
ax[0].set_title("Number of nodes vs alpha")
ax[1].plot(ccp_alphas, depth, marker='o', drawstyle="steps-post")
ax[1].set_xlabel("alpha")
ax[1].set_ylabel("depth of tree")
ax[1].set_title("Depth vs alpha")
fig.tight_layout()<jupyter_output><empty_output><jupyter_text>As alpha increases, more of the tree is pruned, thus creating a decision tree that generalizes better. In this example, setting ccp_alpha=0.015 maximizes the testing accuracy.<jupyter_code>train_scores = [clf.score(X_train, Y_train) for clf in clfs]
test_scores = [clf.score(X_test, Y_test) for clf in clfs]

fig, ax = plt.subplots()
ax.set_xlabel("alpha")
ax.set_ylabel("accuracy")
ax.set_title("Accuracy vs alpha for training and testing sets")
ax.plot(ccp_alphas, train_scores, marker='o', label="train",
        drawstyle="steps-post")
ax.plot(ccp_alphas, test_scores, marker='o', label="test",
        drawstyle="steps-post")
ax.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>try XGBoosting<jupyter_code>from xgboost import XGBClassifier
XGB = XGBClassifier(max_depth=3,                 # Depth of each tree
                            learning_rate=0.1,            # How much to shrink error in each subsequent training. Trade-off with no. estimators.
                            n_estimators=100,             # How many trees to use, the more the better, but decrease learning rate if many used.
                            verbosity=1,                  # If to show more errors or not.
                            objective='binary:logistic',  # Type of target variable.
                            booster='gbtree',             # What to boost. Trees in this case.
                            n_jobs=12,                    # Parallel jobs to run. Set your processor number.
                            gamma=0.001,                  # Minimum loss reduction required to make a further partition on a leaf node of the tree. (Controls growth!)
                            subsample=0.632,              # Subsample ratio. Can set lower
                            colsample_bytree=1,           # Subsample ratio of columns when constructing each tree.
                            colsample_bylevel=1,          # Subsample ratio of columns when constructing each level. 0.33 is similar to random forest.
                            colsample_bynode=1,           # Subsample ratio of columns when constructing each split.
                            reg_alpha=1,                  # Regularizer for first fit. alpha = 1, lambda = 0 is LASSO.
                            reg_lambda=0,                 # Regularizer for first fit.
                            scale_pos_weight=40,           # Balancing of positive and negative weights. 
                                                          # should be set to the positive case balancing weight, or #Goods / #Bads.
                            base_score=0.5,               # Global bias. Set to average of the target rate.
                            random_state=0,       # Seed
                            missing=None                  # How are nulls encoded?
                            )
# Define the parameters. Play with this grid!
param_grid = dict({'n_estimators': [1, 2, 3, 4, 5, 6, 7],
                   'max_depth': [2, 3, 4],
                 'learning_rate' : [0.01, 0.05, 0.1, 0.15]
                  })

# Always a good idea to tune on a reduce sample of the train set, as we will call many functions.
val_train_df = df.sample(frac = 0.05,               # The fraction to extract
                                       random_state = 0,    # The seed.
                                       )
from sklearn.model_selection import GridSearchCV

# Define grid search object.
GridXGB = GridSearchCV(XGB,        # Original XGB. 
                       param_grid,          # Parameter grid
                       cv = 3,              # Number of cross-validation folds.  
                       scoring = 'roc_auc', # How to rank outputs.
                       n_jobs = -1,          # Parallel jobs. -1 is "all you have"
                       refit = False,       # If refit at the end with the best. We'll do it manually.
                       verbose = 1          # If to show what it is doing.
                      )
# Train grid search.
GridXGB.fit(val_train_df[col_kept], val_train_df['Target'])
# save the model
# import pickle
# pkl_filename = 'xgb_cv.pkl'
# with open(pkl_filename, 'wb') as file:
#     pickle.dump(GridXGB, file)
 

# # load the model from disk
# loaded_model = pickle.load(open(filename, 'rb'))
XGB = XGBClassifier(max_depth=GridXGB.best_params_.get('max_depth'), # Depth of each tree
                            learning_rate=GridXGB.best_params_.get('learning_rate'), # How much to shrink error in each subsequent training. Trade-off with no. estimators.
                            n_estimators=GridXGB.best_params_.get('n_estimators'), # How many trees to use, the more the better, but decrease learning rate if many used.
                            verbosity=1,                  # If to show more errors or not.
                            objective='binary:logistic',  # Type of target variable.
                            booster='gbtree',             # What to boost. Trees in this case.
                            n_jobs=12,                    # Parallel jobs to run. Set your processor number.
                            gamma=0.001,                  # Minimum loss reduction required to make a further partition on a leaf node of the tree. (Controls growth!)
                            subsample=0.632,              # Subsample ratio. Can set lower
                            colsample_bytree=1,           # Subsample ratio of columns when constructing each tree.
                            colsample_bylevel=1,          # Subsample ratio of columns when constructing each level. 0.33 is similar to random forest.
                            colsample_bynode=1,           # Subsample ratio of columns when constructing each split.
                            reg_alpha=1,                  # Regularizer for first fit. alpha = 1, lambda = 0 is LASSO.
                            reg_lambda=0,                 # Regularizer for first fit.
                            scale_pos_weight=40,           # Balancing of positive and negative weights.
                            base_score=0.5,               # Global bias. Set to average of the target rate.
                            random_state=0,       # Seed
                            missing=None                  # How are nulls encoded?
                            )
X = df[col_kept]
Y = df['Target']
# Split the train/test
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, shuffle=False)
%%time
# Train over all training data.
XGB.fit(X_train, Y_train)
# Plot variable importance
import seaborn as sns
importances = XGB.feature_importances_
indices = np.argsort(importances)[::-1] 

f, ax = plt.subplots(figsize=(3, 8))
plt.title("Variable Importance - XGBoosting")
sns.set_color_codes("pastel")
sns.barplot(y=[X_train.columns[i] for i in indices], x=importances[indices], 
            label="Total", color="b")
ax.set(ylabel="Variable", xlabel="Variable Importance (Entropy)")
sns.despine(left=True, bottom=True)
from sklearn.metrics import confusion_matrix, roc_curve, roc_auc_score
# Calculate probability
XGBClassTest = XGB.predict(X_test)
xg_probs_test = XGB.predict_proba(X_test)
xg_probs_test = xg_probs_test[:, 1]

# Calculate confusion matrix
confusion_matrix_xgb = confusion_matrix(y_true = Y_test, 
                    y_pred = XGBClassTest)

# Turn matrix to percentages
confusion_matrix_xgb = confusion_matrix_xgb.astype('float') / confusion_matrix_xgb.sum(axis=1)[:, np.newaxis]

# Turn to dataframe
df_cm = pd.DataFrame(
        confusion_matrix_xgb, index=['up', 'down'], columns=['up', 'down'], 
)

# Parameters of the image
figsize = (10,7)
fontsize=14

# Create image
fig = plt.figure(figsize=figsize)
heatmap = sns.heatmap(df_cm, annot=True, fmt='.2f')

# Make it nicer
heatmap.yaxis.set_ticklabels(heatmap.yaxis.get_ticklabels(), rotation=0, 
                             ha='right', fontsize=fontsize)
heatmap.xaxis.set_ticklabels(heatmap.xaxis.get_ticklabels(), rotation=45,
                             ha='right', fontsize=fontsize)

# Add labels
plt.ylabel('True label')
plt.xlabel('Predicted label')

# Plot!
plt.show()
# Calculate the ROC curve points
fpr, tpr, thresholds = roc_curve(Y_test, 
                                 xg_probs_test)

# Save the AUC in a variable to display it. Round it first
auc = np.round(roc_auc_score(y_true = Y_test, 
                             y_score = xg_probs_test),
               decimals = 3)

# Create and show the plot
plt.plot(fpr,tpr,label="AUC - XGBoosting = " + str(auc))
plt.legend(loc=4)
plt.show()<jupyter_output><empty_output><jupyter_text>Stacked LSTM1. Collect stock historical prices.2. Preprocess -- Train and Test3. Build an Stacked LSTM4. Predict test data 5. Predict future 30 days<jupyter_code>plt.plot(df['Adj Close']) 
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler(feature_range=(0,1))
scaled_Adj = scaler.fit_transform(np.array(df['Adj Close']).reshape(-1,1))
scaled_Adj
## Create preprocessed dataset.
training_size = int(len(scaled_Adj)*0.65)
test_size = len(scaled_Adj) - training_size
training_size, test_size
train_LSTM, test_LSTM = scaled_Adj[:training_size, :], scaled_Adj[training_size:len(scaled_Adj), :1]
test_LSTM
test_LSTM.shape
def preprocess_TS(data, time_step=1):
    X, Y = [], []
    for i in range(len(data) - time_step -1):
        a = data[i:(i + time_step), 0]
        X.append(a)
        Y.append(data[i + time_step, 0])
    return np.array(X), np.array(Y)
time_step = 100
X_train_LSTM, Y_train_LSTM = preprocess_TS(train_LSTM, time_step)
X_test_LSTM, Y_test_LSTM = preprocess_TS(test_LSTM, time_step)
X_test_LSTM
# reshape input into [samples, time steps, features] which is required for LSTM
X_train_LSTM = X_train_LSTM.reshape(X_train_LSTM.shape[0], X_train_LSTM.shape[1], 1)
X_test_LSTM = X_test_LSTM.reshape(X_test_LSTM.shape[0], X_test_LSTM.shape[1], 1)
X_test_LSTM
# build a stacked LSTM
import tensorflow
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import LSTM
model = Sequential()
model.add(LSTM(50, return_sequences=True, input_shape=(100,1)))
model.add(LSTM(50, return_sequences=True))
model.add(LSTM(50))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
model.summary()
model.fit(X_train_LSTM, 
          Y_train_LSTM, 
          validation_data=(X_test_LSTM, Y_test_LSTM), 
          epochs=100, 
          batch_size=64, 
          verbose=1)
# predict
train_LSTM_predict = model.predict(X_train_LSTM)
test_LSTM_predict = model.predict(X_test_LSTM)
# transfor back to normal scale
train_LSTM_predict = scaler.inverse_transform(train_LSTM_predict)
test_LSTM_predict = scaler.inverse_transform(test_LSTM_predict)
# Calculate RMSE metrics
from sklearn.metrics import mean_squared_error
np.sqrt(mean_squared_error(Y_train_LSTM, train_LSTM_predict))
# for the test set
np.sqrt(mean_squared_error(Y_test_LSTM, test_LSTM_predict))
### Plotting 
# shift train predictions for plotting
look_back=100
trainPredictPlot = np.empty_like(scaled_Adj)
trainPredictPlot[:, :] = np.nan

trainPredictPlot[look_back:len(train_LSTM_predict)+look_back, :] = train_LSTM_predict
# shift test predictions for plotting
testPredictPlot = np.empty_like(scaled_Adj)
testPredictPlot[:, :] = np.nan
testPredictPlot[len(train_LSTM_predict)+(look_back*2)+1:len(scaled_Adj)-1, :] = test_LSTM_predict
# plot baseline and predictions
plt.plot(scaler.inverse_transform(scaled_Adj))
plt.plot(trainPredictPlot, label="train")
plt.plot(testPredictPlot, label="test")
plt.legend(loc="upper left")
plt.show()

x_input= test_LSTM[len(test_LSTM)-time_step:].reshape(1,-1)
x_input.shape
len(test_LSTM)
temp_input=list(x_input)
temp_input=temp_input[0].tolist()
temp_input
# demonstrate prediction for next 30 days

lst_output=[]
n_steps=100
i=0
while(i<30):
    
    if(len(temp_input)>100):
        #print(temp_input)
        x_input=np.array(temp_input[1:])
        print("{} day input {}".format(i,x_input))
        x_input=x_input.reshape(1,-1)
        x_input = x_input.reshape((1, n_steps, 1))
        #print(x_input)
        yhat = model.predict(x_input, verbose=0)
        print("{} day output {}".format(i,yhat))
        temp_input.extend(yhat[0].tolist())
        temp_input=temp_input[1:]
        #print(temp_input)
        lst_output.extend(yhat.tolist())
        i=i+1
    else:
        x_input = x_input.reshape((1, n_steps,1))
        yhat = model.predict(x_input, verbose=0)
        print(yhat[0])
        temp_input.extend(yhat[0].tolist())
        print(len(temp_input))
        lst_output.extend(yhat.tolist())
        i=i+1
    

print(lst_output)
day_new = np.arange(1,1 + time_step)
day_pred = np.arange(1 + time_step,30 + 1 + time_step)
plt.plot(day_new,scaler.inverse_transform(scaled_Adj[len(scaled_Adj) - 100:]))
plt.plot(day_pred,scaler.inverse_transform(lst_output))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10638924908640811, 'fraction_numerical': 0.015265825769185429, 'mean_word_length': 2.729010989010989, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '454081', 'license': 'Apache-2.0', 'n_tokens_mistral': 5700, 'n_tokens_neox': 5405, 'n_words': 1358, 'path': 'Stock_Movement.ipynb', 'repo_name': 'wjclbj/Time-series-stock-movement-in-decision-tree', 'size': '16966'}