<jupyter_start><jupyter_text>SVD of Minute-Market-Data <NAME> Script Fall 2016<jupyter_code>%reload_ext autoreload
%load_ext autoreload
%autoreload 2
import os
import sys

# add the 'src' directory as one where we can import modules
src_dir = os.path.join(os.getcwd(), os.pardir, 'src')
sys.path.append(src_dir)
%aimport preprocess
from preprocess.process import get_symbol
from preprocess.process import get_symbols_matrix
from preprocess.process import df_to_returns

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os.path
from IPython.core.debugger import Tracer
import scipy as sp
from scipy.linalg import svd
from sklearn.decomposition import PCA
%matplotlib inline<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload<jupyter_text>PreprocessingRead the data in, convert dates to a useful format<jupyter_code>data_loc = "/home/dan/code/market_data/"
symbols = [line.rstrip('\n') for line in open(data_loc+'sp500.txt')]

# Running the SVD on two weeks in 2015, check error on next week
train_start_date = "03/02/2015"
train_end_date = "03/14/2015"

test_start_date = "03/15/2015"
test_end_date = "03/21/2015"

#aapl = get_symbol("AAPL", data_loc, train_start_date, train_end_date)

test_symbols = symbols[400:]
print(test_symbols)

test_mat, syms = get_symbols_matrix(
    test_symbols, data_loc, train_start_date, train_end_date
)
X = test_mat.values.T

# Remove the mean (which roughly removes linear 2-week trend)
Xmean = X.mean(axis=0)
Xd = X-Xmean
#Xd = X

print(Xd.shape)
# Try the SVD
U, S, VT = svd(X)
print(U.shape)
print(S.shape)
print(VT.shape)
print(Xmean.shape)

pca = PCA(svd_solver='randomized')
pca.fit(Xd)

plt.plot(pca.explained_variance_)
plt.show()

market = pca.components_[0, :]
plt.plot( (1+(Xmean)).cumprod() )
plt.show()

# compare to actual market
spy = get_symbol("SPY", data_loc, train_start_date, train_end_date)

plt.plot(spy['Close'].values)
plt.show()
# Bayesian MLP

# Hyperparameters
middle_dim = window_length
encoding_dim = 10
reg = 1e-12
ret_thresh = 1e-2

input_w = Input(shape=(2*window_length,))

in_layer = Dense(
    middle_dim,
    init='normal',
    activation='tanh',
    W_regularizer=l2(reg))(input_w)

mid_layer = Dense(
    encoding_dim,
    init='normal', 
    activation='tanh',
    W_regularizer=l2(reg))(in_layer)

output = Dense(
    1,
    init='normal',
    activation='sigmoid')(mid_layer)

mlp_pred = Model(input=input_w, output=output)
mlp_pred.compile(optimizer='adam', loss='binary_crossentropy')

dm_y = (np.abs(aapl_y[:, 0]) > ret_thresh)*(1 / (2*aapl_y[:,0]))+0.5
dm_val = (np.abs(val_y[:,0]) > ret_thresh)*(1 / (2*val_y[:,0]))+0.5

aapl_f = np.concatenate( (aapl_rets, aapl_vol), axis=1)
val_f = np.concatenate( (val_rets, val_vol), axis=1)

mlp_pred.fit(
    aapl_f, 
    dm_y, 
    batch_size=100,
    nb_epoch=25,
    shuffle=True, 
    validation_data=(val_f, dm_val) 
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12830060934326337, 'fraction_numerical': 0.02742044685172647, 'mean_word_length': 5.314102564102564, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '589792', 'license': 'Apache-2.0', 'n_tokens_mistral': 1235, 'n_tokens_neox': 1160, 'n_words': 254, 'path': 'lambdas/preview/test/data/nb_1200727.ipynb', 'repo_name': 'BearerPipelineTest/quilt', 'size': '2961'}