<jupyter_start><jupyter_text>Feature Engineering for High-Frequency Data  Imports & Settings<jupyter_code>import warnings
warnings.filterwarnings('ignore')
%matplotlib inline

from pathlib import Path

import numpy as np
import pandas as pd

from scipy.stats import spearmanr
import talib

import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style('whitegrid')
idx = pd.IndexSlice<jupyter_output><empty_output><jupyter_text>Data prep We use the 'Trade and Quote' dataset.<jupyter_code>as_path = Path('../data/nasdaq100')
tcols = ['openbartime', 'firsttradetime',
         'highbidtime', 'highasktime', 'hightradetime',
         'lowbidtime', 'lowasktime', 'lowtradetime',
         'closebartime', 'lasttradetime']
drop_cols = ['unknowntickvolume',
             'cancelsize',
             'tradeatcrossorlocked']
keep = ['firsttradeprice', 'hightradeprice', 'lowtradeprice', 'lasttradeprice', 
        'minspread', 'maxspread',
        'volumeweightprice', 'nbboquotecount', 
        'tradeatbid', 'tradeatbidmid', 'tradeatmid', 'tradeatmidask', 'tradeatask', 
        'volume', 'totaltrades', 'finravolume', 
        'finravolumeweightprice', 
        'uptickvolume', 'downtickvolume', 
        'repeatuptickvolume', 'repeatdowntickvolume', 
        'tradetomidvolweight', 'tradetomidvolweightrelative']
columns = {'volumeweightprice'          : 'price',
           'finravolume'                : 'fvolume',
           'finravolumeweightprice'     : 'fprice',
           'uptickvolume'               : 'up',
           'downtickvolume'             : 'down',
           'repeatuptickvolume'         : 'rup',
           'repeatdowntickvolume'       : 'rdown',
           'firsttradeprice'            : 'first',
           'hightradeprice'             : 'high',
           'lowtradeprice'              : 'low',
           'lasttradeprice'             : 'last',
           'nbboquotecount'             : 'nbbo',
           'totaltrades'                : 'ntrades',
           'openbidprice'               : 'obprice',
           'openbidsize'                : 'obsize',
           'openaskprice'               : 'oaprice',
           'openasksize'                : 'oasize',
           'highbidprice'               : 'hbprice',
           'highbidsize'                : 'hbsize',
           'highaskprice'               : 'haprice',
           'highasksize'                : 'hasize',
           'lowbidprice'                : 'lbprice',
           'lowbidsize'                 : 'lbsize',
           'lowaskprice'                : 'laprice',
           'lowasksize'                 : 'lasize',
           'closebidprice'              : 'cbprice',
           'closebidsize'               : 'cbsize',
           'closeaskprice'              : 'caprice',
           'closeasksize'               : 'casize',
           'firsttradesize'             : 'firstsize',
           'hightradesize'              : 'highsize',
           'lowtradesize'               : 'lowsize',
           'lasttradesize'              : 'lastsize',
           'tradetomidvolweight'        : 'volweight',
           'tradetomidvolweightrelative': 'volweightrel'}
parquet_path = as_path / '1min_taq' / 'parquet'
files = list(parquet_path.glob('*.parquet'))
files[:5]
df = pd.read_parquet(files[0]).drop(tcols + drop_cols, axis=1)
df.info()
def data_to_hdf():
    parquet_path = as_path / '1min_taq' / 'parquet'
    files = list(parquet_path.glob('*.parquet'))
    for year in range(2013, 2018):
        print(year)
        data = []
        for f in files:
            if f.stem.startswith(str(year)):
                # print(f.stem, end=' ', flush=True)
                data.append(pd.read_parquet(f).drop(tcols + drop_cols, axis=1))
        data = (pd.concat(data).sort_index()
                .reset_index('ticker')
                .between_time('9:30', '16:00')
                .set_index('ticker', append=True)
                .swaplevel()
                .rename(columns=columns)
                .rename(columns=lambda x: x.replace('tradeat', 'at')))
        print(data.info(null_counts=True))
        data.to_hdf('algoseek.h5', f'data/{year}')
data_to_hdf()
with pd.HDFStore('algoseek.h5') as store:
    df = store['data/2013']
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
MultiIndex: 9853200 entries, ('AAPL', Timestamp('2013-01-02 09:30:00')) to ('YHOO', Timestamp('2013-12-31 16:00:00'))
Data columns (total 43 columns):
 #   Column        Dtype  
---  ------        -----  
 0   obprice       float64
 1   obsize        float64
 2   oaprice       float64
 3   oasize        float64
 4   first         float64
 5   firstsize     float64
 6   hbprice       float64
 7   hbsize        float64
 8   haprice       float64
 9   hasize        float64
 10  high          float64
 11  highsize      float64
 12  lbprice       float64
 13  lbsize        float64
 14  laprice       float64
 15  lasize        float64
 16  low           float64
 17  lowsize       float64
 18  cbprice       float64
 19  cbsize        float64
 20  caprice       float64
 21  casize        float64
 22  last          float64
 23  lastsize      float64
 24  minspread     float64
 25  maxspread     float64
 26  price         float64
 27  nbbo          int64  
 [...]<jupyter_text>Loading Algoseek Data<jupyter_code>ohlcv_cols = ['first', 'high', 'low', 'last', 'price', 'volume']
data_cols = ohlcv_cols + ['up', 'down', 'rup', 'rdown', 'atask', 'atbid']
years = range(2013, 2018)
with pd.HDFStore('algoseek.h5') as store:
    df = (pd.concat([store[f'data/{year}']
                     .loc[:, data_cols] for year in years])
          .sort_index())
df.loc[:, ohlcv_cols[:4]] = df.loc[:, ohlcv_cols[:4]].groupby('ticker').fillna(method='ffill')
df.info(null_counts=True)
df.to_hdf('hf_data.h5', 'data')
df = pd.read_hdf('hf_data.h5', 'data')
# .loc[idx['AAPL', '2013'], :]<jupyter_output><empty_output><jupyter_text>Feature Engineering All of the features above were normalized in a standard fashionby subtracting their means, dividing by their standard deviations, and time-averaging over a recentinterval. In order to obtain a finite state space, features were discretized into bins in multiples ofstandard deviation units<jupyter_code>df = df.sort_index()
df['date'] = pd.to_datetime(df.index.get_level_values('date_time').date)
df.info(null_counts=True)
by_ticker = df.groupby('ticker', group_keys=False)
by_ticker_date = df.groupby(['ticker', 'date'])
data = pd.DataFrame(index=df.index)
data['date'] = pd.factorize(df['date'], sort=True)[0]
data['minute'] = pd.to_timedelta(data.index.get_level_values('date_time').time.astype(str))
data.minute = (data.minute.dt.seconds.sub(data.minute.dt.seconds.min()).div(60).astype(int))<jupyter_output><empty_output><jupyter_text>Lagged Returns<jupyter_code>for t in range(1, 11):
    print(t, end=' ', flush=True)
    data[f'ret{t}min'] = (df
                          .sort_index()
                          .groupby(['ticker', 'date'])
                          .price
                          .pct_change(periods=t, fill_method=None)
                          .shift())<jupyter_output><empty_output><jupyter_text>Forward Returns<jupyter_code>data['fwd1min'] = (data
                   .sort_index()
                   .groupby(['ticker', 'date'])
                   .ret1min
                   .shift(-1))
data.info(null_counts=True)<jupyter_output><empty_output><jupyter_text>Normalized up/downtick volume<jupyter_code>data['rup'] = df.rup.div(df.volume)
data['up'] = df.up.div(df.volume)
data['down'] = df.down.div(df.volume)
data['rdown'] = df.rdown.div(df.volume)
for f in ['up', 'down', 'rup', 'rdown']:
    data[f] = data.groupby(['ticker', 'date'])[f].shift()<jupyter_output><empty_output><jupyter_text>Balance of Power<jupyter_code>data['BOP'] = (by_ticker
               .apply(lambda x: talib.BOP(x['first'],
                                          x.high,
                                          x.low,
                                          x['last'])
                      .shift()))<jupyter_output><empty_output><jupyter_text>Commodity Channel Index<jupyter_code>data['CCI'] = (by_ticker
               .apply(lambda x: talib.CCI(x.high,
                                          x.low,
                                          x['last'],
                                          timeperiod=14).shift()))<jupyter_output><empty_output><jupyter_text>Money Flow Index<jupyter_code>data['MFI'] = (by_ticker
               .apply(lambda x: talib.MFI(x.high,
                                          x.low,
                                          x['last'],
                                          x.volume,
                                          timeperiod=14)
                      .shift()))<jupyter_output><empty_output><jupyter_text>Stochastic RSI<jupyter_code>data['STOCHRSI'] = (by_ticker.apply(lambda x: talib.STOCHRSI(x['last'],
                                                             timeperiod=14,
                                                             fastk_period=14,
                                                             fastd_period=3,
                                                             fastd_matype=0)[0]
                                    .shift()))<jupyter_output><empty_output><jupyter_text>Stochastic<jupyter_code>def compute_stoch(x, fastk_period=14, slowk_period=3, 
                  slowk_matype=0, slowd_period=3, slowd_matype=0):
    slowk, slowd = talib.STOCH(x.high, x.low, x['last'],
                           fastk_period=fastk_period,
                           slowk_period=slowk_period,
                           slowk_matype=slowk_matype,
                           slowd_period=slowd_period,
                           slowd_matype=slowd_matype)
    return (slowd/slowk-1).shift()
data['STOCH'] = by_ticker.apply(compute_stoch).replace((np.inf, -np.inf), np.nan)<jupyter_output><empty_output><jupyter_text>Transaction Volume by price point<jupyter_code>data['trades_bid_ask'] = df.atask.sub(df.atbid).div(df.volume).replace((np.inf, -np.inf), np.nan)
data['trades_bid_ask'] = data.groupby(['ticker', 'date']).trades_bid_ask.shift()<jupyter_output><empty_output><jupyter_text>Evaluate features<jupyter_code>features = ['ret1min', 'ret2min', 'ret3min', 'ret4min', 'ret5min', 
            'ret6min', 'ret7min', 'ret8min', 'ret9min', 'ret10min',
            'rup', 'up', 'down', 'rdown', 
            'BOP', 'CCI', 'MFI', 'STOCHRSI', 'STOCH', 
            'trades_bid_ask']
ic = {}
for feature in features:
    print(feature)
    df_ = data[['fwd1min', feature]].dropna()
    ic[feature] = spearmanr(df_.fwd1min, df_[feature])
ic = pd.Series(ic).apply(pd.Series)
ic.columns = ['IC', 'p-value']    
ic.sort_values('IC')
ic.index = ic.index.map(str.upper)
ax = ic['IC'].sort_values(ascending=False).mul(100).plot.bar(figsize=(14, 4), 
                                                        title='Information Coeficient for HF Features (1-min forward returns)',
                                                       rot=0)
ax.set_ylabel('Information Coefficient')
plt.tight_layout()
plt.savefig('figures/hft_ic', dpi=300);
ic.sort_values('IC').to_csv('hf_ic.csv')<jupyter_output><empty_output><jupyter_text>Store results<jupyter_code>data.drop(['date', 'up', 'down'], axis=1).to_hdf('hf_data.h5', 'model_data')
data.info(null_counts=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13479789103690684, 'fraction_numerical': 0.02100175746924429, 'mean_word_length': 2.053662463107057, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 33, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '449226', 'license': 'MIT', 'n_tokens_mistral': 3821, 'n_tokens_neox': 3598, 'n_words': 612, 'path': '12_gradient_boosting_machines/10_intraday_features.ipynb', 'repo_name': 'driscolljt/Machine-Learning-for-Algorithmic-Trading-Second-Edition_Original', 'size': '11380'}