<jupyter_start><jupyter_text>WQU Capstone project - Short-term trading strategy on G10 currencies Notebook five - Putting previous steps together and Feature Engineering* <NAME> - <EMAIL>* <NAME> -  <EMAIL>* <NAME> - <EMAIL> Jun-Aug-2020 After these steps the data should be ready for classification<jupyter_code>import sys
import os
#insert you own path or use relative path
path_to_project = os.path.realpath('..') # r'C:\WQU\Capstone\Working_files'
sys.path.append(path_to_project)
import pandas as pd
import numpy as np
import datetime as dt
from pandas.tseries.offsets import BMonthEnd
from multiprocessing import cpu_count
#from statsmodels.tsa.stattools import adfuller
from tqdm import tqdm
import warnings
warnings.filterwarnings('ignore')
from WQUcapstoneCode.sampling import sampling
from WQUcapstoneCode.labeling import labeling
from WQUcapstoneCode.technical import technical
from WQUcapstoneCode.fracdif.fracdif import frac_diff_ffd
import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
plt.style.use('seaborn-talk')
plt.style.use('bmh')<jupyter_output><empty_output><jupyter_text>Settings<jupyter_code>tickers = ['AUD/USD','AUD/CAD','AUD/JPY','EUR/USD','GBP/USD','NZD/USD','USD/CAD', 'USD/JPY']
max_holding_period = 999 #days 
ticks_multiplyer = 1. #arbitrary
min_ret_target_vol_multiplier = 0.7 #
cpus = cpu_count()-1
d = 0.35 #fracdiff parameter. 1=simple first order differencing
#we can use ADF to find non-stationary features, however, 
#fractionally differentiated featured may hold useful information for further analysis
#therefore, we'll use default list of features to apply fracDiff
non_stationary_feat = {'price','fast','slow','average','upper_band','lower_band','tenka_sen','kijun_sen','senkou_span_a','senkou_span_b'}<jupyter_output><empty_output><jupyter_text>Load interest rate data<jupyter_code># Import interbank interest rate data
# --------------------------------------------------------------------------------
input_path_ir_data = os.path.join(path_to_project, 'input_data', 'interbank_IR_3m.xlsx')
ir = pd.read_excel(input_path_ir_data,sheet_name = 'consolidatedData')
ir.index =[dt.datetime.strptime(date, '%Y-%m-%d %H:%M:%S') for date in ir.date]<jupyter_output><empty_output><jupyter_text>Data preparation<jupyter_code>offset = BMonthEnd()

for ticker in tickers:
    
    # Print ticket the loop is running
    print (ticker+": Data preparation in progres...")
    
    # Import FX spot data
    # --------------------------------------------------------------------------------
    input_path = os.path.join(path_to_project, 'input_data', ''.join(ticker.split('/')) + '.csv')
    pair = pd.read_csv(input_path)
    pair.index =[dt.datetime.strptime(date, '%Y-%m-%d %H:%M:%S') for date in pair.date]
    pair = pair.drop(columns=['date'])
    
    # Assign CCY1 and CCY2 from ticker
    # --------------------------------------------------------------------------------
    CCY1 = ticker[:3]
    CCY2 = ticker[-3:]
    
    # calculate interest rate differentials
    # --------------------------------------------------------------------------------
    ir_d1 = pd.DataFrame(index=ir.index, columns=['ir_d1'])
    ir_d1['ir_d1']= ir[CCY1] - ir[CCY2]
        
    m_ticks = ticks_multiplyer * pair.tickqty.sum()/pair.shape[0]
    
    #print('Sampling')
    # --------------------------------------------------------------------------------
    tick_df = sampling.sampled_bar_df(pair, 'tickqty',m_ticks)
    
    #print('Labeling')   
    # --------------------------------------------------------------------------------    
    dailyVol = labeling.getDailyVol(tick_df.bidclose) 
    dailyVol.name = 'volatility'
    close = tick_df[['bidclose','askclose']]
    close.columns = ['bid','ask']  
    tEvents = labeling.getTEvents(close,h=dailyVol.mean())
    t1 = labeling.addVerticalBarrier(tEvents, close, numDays=max_holding_period)
    ptsl = [1, 1] #symmetric take-profit and stop-loss
    target = dailyVol
    # select minRet
    minRet = dailyVol.mean()*min_ret_target_vol_multiplier
    close = (close['bid'] + close['ask']) / 2 #to simplify we'll work with the mid price
    events = labeling.getEvents(close, tEvents, ptsl, target, minRet, cpus, t1=t1)
    labels = labeling.getBins(events, close)
    labels = labeling.dropLabels(labels)
    
    #print('Calculating technical features')
    ema = technical.EMA(close)
    bb = technical.BollingerBands(close)
    cci = technical.CCI(close)
    so = technical.Stochastic(close)
    wr = technical.wr(close)
    ic = technical.Ichimoku(close)
    rsi = technical.RSI(close)
    
    lagged_px = pd.concat([close.shift(1), close.shift(2),close.shift(12)],axis=1)
    lagged_px.columns = ['T-1','T-2','T-12']
    lagged_returns = pd.DataFrame(np.diff(np.log(lagged_px), axis=0), 
                                  index = lagged_px.index[1:], 
                                  columns=[c+'_1per_rtn' for c in lagged_px])
    period_returns = pd.concat([np.log(close/close.shift(1)), 
                                np.log(close/close.shift(1)), 
                                np.log(close/close.shift(1))],axis=1)
    period_returns.columns = ['T-1_rtn','T-2_rtn','T-12_rtn']
    lags = pd.concat([technical.rolling_autocorr(close,lag=1),
                      technical.rolling_autocorr(close,lag=2),
                      technical.rolling_autocorr(close,lag=4),
                      technical.rolling_autocorr(close,lag=6)], axis=1) #2,4,6 were found to be correlated earlier
    
    ema.data = ema.data.rename(columns={'side':'ema_side'})
    bb.data = bb.data.rename(columns={'side':'bb_side'})
    so.data = so.data.rename(columns={'side':'so_side'})
    cci.data = cci.data.rename(columns={'side':'cci_side'})
    wr.data = wr.data.rename(columns={'side':'wr_side'})
    ic.data = ic.data.rename(columns={'side':'ic_side'})
    rsi.data = rsi.data.rename(columns={'side':'rsi_side'})
    feat = lambda x: [col for col in x.columns if col != 'price'] 
    features = pd.concat([ema(), 
                          bb()[feat], 
                          so()[feat], 
                          cci()[feat],
                          wr()[feat],
                          ic()[feat], 
                          rsi()[feat], 
                          lagged_px, 
                          lags,
                          lagged_returns, 
                          period_returns,
                          ir_d1], axis=1)
    #day of the week and month may be useful features as well because of rebalancing flows
    #we one-hot-encode them
    features = pd.concat([features,
                          pd.get_dummies(features.index.day_name()).set_index(features.index),
                          pd.get_dummies(features.index.month_name()).set_index(features.index)], axis=1)
    #features['day'], features['month'] = features.index.dayofweek, features.index.month
    features['EOM']=features.index.map(lambda x: 1 if (offset.rollforward(x).day==x.day) else 0)
    #features['EOQ']=features.index.map(lambda x: 1 if ((offset.rollforward(x).day==x.day) and (x.month in (3,6,9,12))) else 0)
    #print('Applying fracDiff')
    df = pd.concat([frac_diff_ffd(pd.DataFrame(features[c].dropna()), diff_amt=d, thresh=1e-5)  for c in tqdm(non_stationary_feat)], axis = 1)
    df.columns = [f'{c}_frdif' for c in df]
    
    result = pd.concat([features,df,dailyVol, labels], axis = 1).dropna()
    preprocessed_path= os.path.join(path_to_project, 'preprocessed_data', ''.join(ticker.split('/')) + '_feat.csv')
    result.to_csv(preprocessed_path)<jupyter_output>AUD/USD: Data preparation in progres...
{'fraction_non_alphanumeric': 0.19126683226565563, 'fraction_numerical': 0.01058961955811217, 'mean_word_length': 2.543307086614173, 'pattern_counts': {'":': 1, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '302131', 'license': 'MIT', 'n_tokens_mistral': 2525, 'n_tokens_neox': 2417, 'n_words': 475, 'path': 'notebooks/WQU Capstone - Notebook 5 - Feature Engineering.ipynb', 'repo_name': 'dhruva1/capstone', 'size': '7720'}