<jupyter_start><jupyter_text>Bloomblerg Japan Hackathon 2018/09/01The goal is to predict cryptocurrency price in the future. If possible, predict the timing to buy and sell.Our team decided to use ensemble of different models, semantic text for hype in twitter discussion, technical analysis of historical price and influencse of other finacial markets like stock market and gold market, etc.This notebook tries to implement the last idea, uses recent Bitcoin price, trade volume, NASDAQ index, OIL ETF index, GOLD ETF index and companies like BLK and BTCS as inputs to predict the BTC/USD price of the next day, given limited amount of time in this hackathon. Of course, the model can be extended to include more factors.<jupyter_code>from math import sqrt
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error, mean_absolute_error
from keras.models import Sequential
from keras.layers import Dense, LSTM<jupyter_output>/Users/sushi/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters
Using TensorFlow backend.<jupyter_text>Get data from 2017/01/01 to 2017/11/09Our team want to use data from 2017/09/01 to 2017/11/09 as testing data.<jupyter_code>import glob
files = glob.glob("findex_data/*.csv")
print(files)
flie_list = []
for f in files:
    _df = pd.read_csv(f, header=0)
    flie_list.append(_df[['Open','Volume']])
df = pd.concat(flie_list, axis=1, sort=False)
index_names = ['BTC','BTC_V','BTCS','BTCS_V','NASDAQ','NASDAQ_V','OIL','OIL_V','GOLD','GOLD_V','BLK','BLK_V']
df.columns= index_names
df.head()
number_of_features = 12
plt.figure(figsize=(20,30))
for i in range(0, number_of_features):
    plt.subplot(number_of_features, 1, i+1)
    plt.plot(df.values[:, i])
    plt.title(df.columns[i], loc='right')
plt.show()
scaler = MinMaxScaler(feature_range=(0, 1))
scaled = scaler.fit_transform(df.values)
scaled_df = pd.DataFrame(scaled, columns=index_names)
scaled_df.head()
#goal: use previous 10 steps to predict a new step
number_of_steps = 10
stepped_df_list = [scaled_df,]
for i in range(1, number_of_steps+1):
    one_step_df = scaled_df.shift(-i)
    one_step_df.columns = ["{}+{}d".format(x, i) for x in scaled_df.columns]
    stepped_df_list.append(one_step_df)
stepped_df = pd.concat(stepped_df_list, axis=1)
#drop first number_of_steps of rows
stepped_df = stepped_df.iloc[:-number_of_steps, :]
stepped_df.head()
#train test split
#train period 2017/01/01 to 2017/08/31, data length = 168
#test period 2017/09/01 to 2017/11/10
number_training_rows = 168
values = stepped_df.values
train = values[:number_training_rows-number_of_steps, :]
test = values[number_training_rows-number_of_steps:, :]
train_X, train_y = train[:, :-number_of_features], train[:, -number_of_features]
test_X, test_y = test[:, :-number_of_features], test[:, -number_of_features]
# reshape to (samples, steps, features)
step_train_X = train_X.reshape((train_X.shape[0], number_of_steps, number_of_features))
step_test_X = test_X.reshape((test_X.shape[0], number_of_steps, number_of_features))
print(step_train_X.shape, train_y.shape, step_test_X.shape, test_y.shape)
#LSTM model
model = Sequential()
model.add(LSTM(30, input_shape=(step_train_X.shape[1], step_train_X.shape[2])))
model.add(Dense(1))
model.compile(loss='mae', optimizer='adam')
history = model.fit(step_train_X, train_y, epochs=100, batch_size=10, validation_data=(step_test_X, test_y), shuffle=False)
plt.plot(history.history['loss'], label='train')
plt.plot(history.history['val_loss'], label='test')
plt.legend()
plt.show()
# predict
y_pred = model.predict(step_test_X)
# mean absolute error
print("Prediction MAE: {}".format(mean_absolute_error(test_y, y_pred)))
# compare with original price
btc_df = pd.read_csv('findex_data/B.csv', usecols=['Date', 'Open'])
pred_df = pd.DataFrame(btc_df.values[number_training_rows:,:])
pred_df.columns = ['Date', 'Actual Price']
def inverse_scale(y):
    y = np.concatenate((y, test_X[:, -(number_of_features-1):]), axis=1)
    y = scaler.inverse_transform(y)
    return y[:,0]
# inverse back to original prices
iy_pred = inverse_scale(y_pred)
pred_df['Predict'] = iy_pred
pred_df
ax = pred_df.plot(xticks=pred_df.index, y=['Actual Price', 'Predict'], figsize=(20,10), grid=True)
ax.set_xticklabels(pred_df.Date, rotation=90)
plt.legend(loc='upper left')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12580368624089155, 'fraction_numerical': 0.027218174024860695, 'mean_word_length': 6.407936507936508, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '91704', 'license': 'MIT', 'n_tokens_mistral': 1739, 'n_tokens_neox': 1632, 'n_words': 417, 'path': 'financial_market_influence.ipynb', 'repo_name': 'suzana-ilic/crypto_stock_market_prediction', 'size': '4666'}