<jupyter_start><jupyter_text>Financial Forecast Demo<jupyter_code>import os
import sys
import uuid
import time
import json
from pathlib import Path

##### Import my module build from python file
module_path = str(Path.cwd() / "py")
debug_data_path = str(Path.cwd() / "debug_data")

if module_path not in sys.path:
    sys.path.append(module_path)
import build as build
import utils
from utils import get_module_version
log = utils.setup_logging(__name__)

##### Import common machine learning modules
import numpy as np
import pandas as pd
import pickle
log.debug('numpy version: %s' % np.__version__)

##### Import sklearn
import sklearn
from sklearn.linear_model import LinearRegression
from sklearn import preprocessing, svm 
from sklearn.model_selection import train_test_split
log.debug('sklearn version: %s' % sklearn.__version__)
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler<jupyter_output>2021-03-29 23:29:01,357 - utils - DEBUG - test debug logging
2021-03-29 23:29:01,364 - __main__ - DEBUG - numpy version: 1.20.2
2021-03-29 23:29:01,441 - __main__ - DEBUG - sklearn version: 0.23.0<jupyter_text>Model parameters.<jupyter_code>MODEL_WINDOW_SIZE = 60
# Sklearn test_size
SKLEARN_TEST_SIZE = 0.20<jupyter_output><empty_output><jupyter_text>All credentials for Watson Machine Learning, Watson OpenScale and its associated DataMart<jupyter_code>model_path = build.get_model_base_path()
log.debug("Model Base Path: " + model_path)
#debug_data_csv = 'debug_data/EURUSD_M1_202103150000_202103251441.csv'

#debug_data_csv = 'debug_data/EURUSD_M1_202102250000_202103251443.csv'

debug_data_csv = 'debug_data/EURUSD_M1_202012150519_202103251444.csv'
def get_training_float_data_from_exported_csv() :
    history_dataset = pd.read_csv(debug_data_csv, sep='\t')
    #history_dataset['DATE_TIME'] = history_dataset['<DATE>'] + ' ' + history_dataset['<TIME>']
    values = history_dataset['<CLOSE>'].values
    log.debug(type(values.tolist()))
    return values.tolist()
def get_training_float_data_from_json() :
    debug_data_json_path = debug_data_path + '/input_data.json'
    with open(debug_data_json_path) as json_file:
        input_data = json.load(json_file)
    log.debug(type(input_data['Data']))
    float_data = [float(s) for s in input_data['Data']]
float_data = get_training_float_data_from_exported_csv()
X, y = build.get_feature_label_sklearn(float_data, default_window_size=MODEL_WINDOW_SIZE)

print("====" + str(len(X[-1])))
print("====")
# Splitting the data into training and testing data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = SKLEARN_TEST_SIZE, random_state=None, shuffle=False) 

log.debug(type(X_test[0]))
print(X_test[0])
print(pd.DataFrame(X_test))
fields = ['f' + str(i) for i in range(MODEL_WINDOW_SIZE)]
log.debug(fields)

#print(len(X[-1]))

#X_np = np.array([np.array(xi) for xi in X])

training_data_df = pd.DataFrame(X, columns=fields)
log.debug(len(y))
training_data_df['l1'] = y

training_data_file_name="model_data.csv"

training_data_df.head(10)
training_data_df.to_csv(training_data_file_name, index = False, header=True)
scaler = MinMaxScaler()

model = LinearRegression()

pipeline = Pipeline([('scaler', scaler), ('lr', model)])
type(pipeline)

pipeline.fit(X_train, y_train)

# model.fit(X_train, y_train)

log.debug("Linear regression score: %f" % model.score(X_test, y_test)) 
log.debug("Coeffience: ")
log.debug(model.coef_)
log.debug("Intercept: %f" % model.intercept_)

# Save model
sklearn_model_file_pickle = 'saved_model/ff_sklearn_pickle.pkl'
build.save_sklearn_model(model, sklearn_model_file_pickle)

# Load model
model = build.load_sklearn_model(sklearn_model_file_pickle)

# Predict X_test, all test samples
y#_pred = model.predict(X_test)
y_pred = pipeline.predict(X_test)
df = pd.DataFrame({'Actual': y_test, 'Predicted': y_pred})
df
from sklearn import metrics
print('Mean Absolute Error:', metrics.mean_absolute_error(y_test, y_pred))
print('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))
print('R2 score:', metrics.r2_score(y_test, y_pred))<jupyter_output>Mean Absolute Error: 8.40318366615625e-05
Mean Squared Error: 1.6048633406592367e-08
Root Mean Squared Error: 0.00012668320096442294
R2 score: 0.9990856998275862
{'fraction_non_alphanumeric': 0.11835529304861427, 'fraction_numerical': 0.050658791458427985, 'mean_word_length': 6.8625, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '374085', 'license': 'Apache-2.0', 'n_tokens_mistral': 1751, 'n_tokens_neox': 1600, 'n_words': 324, 'path': 'model_test.ipynb', 'repo_name': 'pholas/forex-forecast', 'size': '4402'}