<jupyter_start><jupyter_text>Imitation Learning for Portfolio Management<jupyter_code>from model.supervised.imitation_optimal_action import *
from utils.data import create_optimal_imitation_dataset, create_imitation_dataset, read_stock_history, normalize
from __future__ import print_function
import random
%matplotlib inline
%load_ext autoreload
%autoreload 2
# dataset
history, abbreviation = read_stock_history(filepath='utils/datasets/stocks_history_target.h5')
history = history[:, :, :4]


# 16 stocks are all involved. We choose first 3 years as training data and last 2 years as testing data.
num_training_time = 1095
target_stocks = abbreviation
target_history = np.empty(shape=(len(target_stocks), num_training_time, history.shape[2]))

for i, stock in enumerate(target_stocks):
    target_history[i] = history[abbreviation.index(stock), :num_training_time, :]

# test on 3 never seen stocks
test_stocks = abbreviation
test_history = np.empty(shape=(len(test_stocks), history.shape[1] - num_training_time, history.shape[2]))
for i, stock in enumerate(test_stocks):
    test_history[i] = history[abbreviation.index(stock), num_training_time:, :]<jupyter_output><empty_output><jupyter_text>Train optimal action given future observation<jupyter_code># build optimal model given future model
nb_classes = len(target_stocks) + 1
optimal_given_future_model = create_network_given_future(nb_classes)
# run this cell to train optimal action given future model
train_optimal_action_given_future_obs(optimal_given_future_model, target_history, target_stocks)<jupyter_output><empty_output><jupyter_text>Test learning from optimal action using future data: sanity check<jupyter_code>(X_test, y_test), (_, _) = create_optimal_imitation_dataset(test_history)
Y_test = np_utils.to_categorical(y_test, nb_classes)
loss, acc = optimal_given_future_model.evaluate(X_test, Y_test)
print('Testing result: loss - {}, accuracy - {}'.format(loss, acc))<jupyter_output>584/584 [==============================] - 0s 110us/step
Testing result: loss - 0.781989398068, accuracy - 0.772260273973<jupyter_text>Start to play in an environment with stocks we have never seen before<jupyter_code>from environment.portfolio import PortfolioEnv
env = PortfolioEnv(test_history, test_stocks, steps=365)
# buy and sell only 1 stock
done = False
observation, info = env.reset()
ground_truth_obs = info['next_obs']
close_open_ratio = np.transpose(ground_truth_obs[:, :, 3] / ground_truth_obs[:, :, 0])
close_open_ratio = normalize(close_open_ratio)
while not done:
    action = np.zeros((nb_classes,))
    current_action_index = optimal_given_future_model.predict_classes(close_open_ratio, verbose=False)
    action[current_action_index] = 1.0
    observation, reward, done, info = env.step(action)
    ground_truth_obs = info['next_obs']
    close_open_ratio = np.transpose(ground_truth_obs[:, :, 3] / ground_truth_obs[:, :, 0])
    close_open_ratio = normalize(close_open_ratio)
env.render()<jupyter_output><empty_output><jupyter_text>It turns out that using probability distribution sometimes yield higher return<jupyter_code># buy and sell according to the probability, not the label
done = False
observation, info = env.reset()
ground_truth_obs = info['next_obs']
close_open_ratio = np.transpose(ground_truth_obs[:, :, 3] / ground_truth_obs[:, :, 0])
close_open_ratio = normalize(close_open_ratio)
while not done:
    current_action = optimal_given_future_model.predict(close_open_ratio, verbose=False)
    current_action = np.squeeze(current_action, axis=0)
    observation, reward, done, info = env.step(current_action)
    ground_truth_obs = info['next_obs']
    close_open_ratio = np.transpose(ground_truth_obs[:, :, 3] / ground_truth_obs[:, :, 0])
    close_open_ratio = normalize(close_open_ratio)
env.render()<jupyter_output><empty_output><jupyter_text>Train optimal action given history observation<jupyter_code># we need to test different window length
window_length = 3
# create model
optimal_given_past_model = create_network_give_past(nb_classes, window_length)
# run this cell to train the model. 
# For 3 stocks, it would take 200 epoches to converge to around 45% validation accuracy
# For 16 stocks, the validation accuracy is around 15%.
train_optimal_action_given_history_obs(optimal_given_past_model, target_history, target_stocks, window_length)<jupyter_output><empty_output><jupyter_text>Test optimal action given history observation<jupyter_code>(X_test, y_test), (_, _) = create_imitation_dataset(test_history, window_length=window_length)
Y_test = np_utils.to_categorical(y_test, nb_classes)
# increase a dimension of X_test for CNN
X_test = np.expand_dims(X_test, axis=-1)
loss, acc = optimal_given_past_model.evaluate(X_test, Y_test)
print('Testing result: loss - {}, accuracy - {}'.format(loss, acc))
env = PortfolioEnv(test_history, test_stocks, window_length=window_length, steps=365)
# buy and sell only 1 stock
done = False
observation, _ = env.reset()
close_open_ratio = observation[:, :, 3] / observation[:, :, 0]
close_open_ratio = normalize(close_open_ratio)
while not done:
    action = np.zeros((nb_classes,))
    close_open_ratio = np.expand_dims(close_open_ratio, axis=0)
    close_open_ratio = np.expand_dims(close_open_ratio, axis=-1)
    current_action_index = optimal_given_past_model.predict_classes(close_open_ratio, verbose=False)
    action[current_action_index] = 1.0
    observation, reward, done, _ = env.step(action)
    close_open_ratio = observation[:, :, 3] / observation[:, :, 0]
    close_open_ratio = normalize(close_open_ratio)
env.render()
# buy and sell according to the probability, not the label
done = False
observation, _ = env.reset()
close_open_ratio = observation[:, :, 3] / observation[:, :, 0]
while not done:
    close_open_ratio = np.expand_dims(close_open_ratio, axis=0)
    close_open_ratio = np.expand_dims(close_open_ratio, axis=-1)
    close_open_ratio = normalize(close_open_ratio)
    current_action = optimal_given_past_model.predict(close_open_ratio, verbose=False)
    current_action = np.squeeze(current_action, axis=0)
    observation, reward, done, _ = env.step(current_action)
    close_open_ratio = observation[:, :, 3] / observation[:, :, 0]
env.render()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1046902513206339, 'fraction_numerical': 0.015687530014406915, 'mean_word_length': 6.969387755102041, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '490411', 'license': 'MIT', 'n_tokens_mistral': 2129, 'n_tokens_neox': 2014, 'n_words': 455, 'path': 'src/imitation_learning_experiments.ipynb', 'repo_name': 'satyenataagyaa/drl-portfolio-management', 'size': '6247'}