<jupyter_start><jupyter_text>Japanese Equity Data - Feature Engineering  Imports & Settings<jupyter_code>import warnings
warnings.filterwarnings('ignore')
%matplotlib inline

from pathlib import Path

import numpy as np
import pandas as pd
import talib

import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style('white')
idx = pd.IndexSlice<jupyter_output><empty_output><jupyter_text>Get Data  Stooq Japanese Equity data 2014-2019<jupyter_code>DATA_DIR = Path('..', 'data')
prices = (pd.read_hdf(DATA_DIR / 'assets.h5', 'stooq/jp/tse/stocks/prices')
          .loc[idx[:, '2010': '2019'], :]
          .loc[lambda df: ~df.index.duplicated(), :])
prices.info(show_counts=True)
before = len(prices.index.unique('ticker').unique())<jupyter_output><empty_output><jupyter_text>Remove symbols with missing values<jupyter_code>prices = (prices.unstack('ticker')
        .sort_index()
        .ffill(limit=5)
        .dropna(axis=1)
        .stack('ticker')
        .swaplevel())
prices.info(show_counts=True)
after = len(prices.index.unique('ticker').unique())
print(f'Before: {before:,.0f} after: {after:,.0f}')<jupyter_output>Before: 3,664 after: 2,278<jupyter_text>Keep most traded symbols<jupyter_code>dv = prices.close.mul(prices.volume)
keep = dv.groupby('ticker').median().nlargest(1000).index.tolist()
prices = prices.loc[idx[keep, :], :]
prices.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
MultiIndex: 2449000 entries, ('9984.JP', Timestamp('2010-01-04 00:00:00')) to ('8107.JP', Timestamp('2019-12-30 00:00:00'))
Data columns (total 5 columns):
 #   Column  Dtype  
---  ------  -----  
 0   open    float64
 1   high    float64
 2   low     float64
 3   close   float64
 4   volume  float64
dtypes: float64(5)
memory usage: 103.0+ MB<jupyter_text>Feature Engineering  Compute period returns<jupyter_code>intervals = [1, 5, 10, 21, 63]
returns = []
by_ticker = prices.groupby(level='ticker').close
for t in intervals:
    returns.append(by_ticker.pct_change(t).to_frame(f'ret_{t}'))
returns = pd.concat(returns, axis=1)
returns.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
MultiIndex: 2449000 entries, ('9984.JP', Timestamp('2010-01-04 00:00:00')) to ('8107.JP', Timestamp('2019-12-30 00:00:00'))
Data columns (total 5 columns):
 #   Column  Dtype  
---  ------  -----  
 0   ret_1   float64
 1   ret_5   float64
 2   ret_10  float64
 3   ret_21  float64
 4   ret_63  float64
dtypes: float64(5)
memory usage: 103.0+ MB<jupyter_text>Remove outliers<jupyter_code>max_ret_by_sym = returns.groupby(level='ticker').max()
percentiles = [0.001, .005, .01, .025, .05, .1]
percentiles += [1-p for p in percentiles]
max_ret_by_sym.describe(percentiles=sorted(percentiles)[6:])
quantiles = max_ret_by_sym.quantile(.95)
to_drop = []
for ret, q in quantiles.items():
    to_drop.extend(max_ret_by_sym[max_ret_by_sym[ret]>q].index.tolist()) 
to_drop = pd.Series(to_drop).value_counts()
to_drop = to_drop[to_drop > 1].index.tolist()
len(to_drop)
prices = prices.drop(to_drop, level='ticker')
prices.info(show_counts=True)<jupyter_output><class 'pandas.core.frame.DataFrame'>
MultiIndex: 2304509 entries, ('9984.JP', Timestamp('2010-01-04 00:00:00')) to ('8107.JP', Timestamp('2019-12-30 00:00:00'))
Data columns (total 5 columns):
 #   Column  Dtype  
---  ------  -----  
 0   open    float64
 1   high    float64
 2   low     float64
 3   close   float64
 4   volume  float64
dtypes: float64(5)
memory usage: 96.9+ MB<jupyter_text>Calculate relative return percentiles<jupyter_code>returns = []
by_sym = prices.groupby(level='ticker').close
for t in intervals:
    ret = by_sym.pct_change(t)
    rel_perc = (ret.groupby(level='date')
             .apply(lambda x: pd.qcut(x, q=20, labels=False, duplicates='drop')))
    returns.extend([ret.to_frame(f'ret_{t}'), rel_perc.to_frame(f'ret_rel_perc_{t}')])
returns = pd.concat(returns, axis=1)<jupyter_output><empty_output><jupyter_text>Technical Indicators  Percentage Price Oscillator<jupyter_code>ppo = prices.groupby(level='ticker').close.apply(talib.PPO).to_frame('PPO')<jupyter_output><empty_output><jupyter_text>Normalized Average True Range<jupyter_code>natr = prices.groupby(level='ticker', group_keys=False).apply(lambda x: talib.NATR(x.high, x.low, x.close)).to_frame('NATR')<jupyter_output><empty_output><jupyter_text>Relative Strength Indicator<jupyter_code>rsi = prices.groupby(level='ticker').close.apply(talib.RSI).to_frame('RSI')<jupyter_output><empty_output><jupyter_text>Bollinger Bands<jupyter_code>def get_bollinger(x):
    u, m, l = talib.BBANDS(x)
    return pd.DataFrame({'u': u, 'm': m, 'l': l})
bbands = prices.groupby(level='ticker').close.apply(get_bollinger)<jupyter_output><empty_output><jupyter_text>Combine Features<jupyter_code>data = pd.concat([prices, returns, ppo, natr, rsi, bbands], axis=1)
data['bbl'] = data.close.div(data.l)
data['bbu'] = data.u.div(data.close)
data = data.drop(['u', 'm', 'l'], axis=1)
data.bbu.corr(data.bbl, method='spearman')<jupyter_output><empty_output><jupyter_text>Plot Indicators for randomly sample ticker<jupyter_code>indicators = ['close', 'bbl', 'bbu', 'PPO', 'NATR', 'RSI']
ticker = np.random.choice(data.index.get_level_values('ticker'))
(data.loc[idx[ticker, :], indicators].reset_index('ticker', drop=True)
 .plot(lw=1, subplots=True, figsize=(16, 10), title=indicators, layout=(3, 2), legend=False))
plt.suptitle(ticker, fontsize=14)
sns.despine()
plt.tight_layout()
plt.subplots_adjust(top=.95)
data = data.drop(prices.columns, axis=1)<jupyter_output><empty_output><jupyter_text>Create time period indicators<jupyter_code>dates = data.index.get_level_values('date')
data['weekday'] = dates.weekday
data['month'] = dates.month
data['year'] = dates.year<jupyter_output><empty_output><jupyter_text>Compute forward returns<jupyter_code>outcomes = []
by_ticker = data.groupby('ticker')
for t in intervals:
    k = f'fwd_ret_{t:02}'
    outcomes.append(k)
    data[k] = by_ticker[f'ret_{t}'].shift(-t)
data.info(null_counts=True)
data.to_hdf('data.h5', 'stooq/japan/equities')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1645465058648604, 'fraction_numerical': 0.04741450520403106, 'mean_word_length': 5.652747252747253, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '257346', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2550, 'n_tokens_neox': 2335, 'n_words': 392, 'path': 'ml4trading-2ed/11_decision_trees_random_forests/04_japanese_equity_features.ipynb', 'repo_name': 'JennEYoon/SEC-data', 'size': '6053'}