<jupyter_start><jupyter_text>Get returns for equities with highest market cap<jupyter_code>idx = pd.IndexSlice
with pd.HDFStore('../../data/assets.h5') as store:
    stocks = store['us_equities/stocks'].marketcap.nlargest(500)
    returns = (store['quandl/wiki/prices']
               .loc[idx['2010': '2018', stocks.index], 'adj_close']
               .unstack('ticker')
               .pct_change())
returns.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
DatetimeIndex: 2072 entries, 2010-01-04 to 2018-03-27
Columns: 351 entries, A to ZTS
dtypes: float64(351)
memory usage: 5.6 MB<jupyter_text>Winsorize & standardize returns<jupyter_code>returns = returns.clip(lower=returns.quantile(q=.025),
                       upper=returns.quantile(q=.975),
                       axis=1)<jupyter_output><empty_output><jupyter_text>Impute missing values<jupyter_code>returns = returns.dropna(thresh=int(returns.shape[0] * .95), axis=1)
returns = returns.dropna(thresh=int(returns.shape[1] * .95))
returns.info()
daily_avg = returns.mean(1)
returns = returns.apply(lambda x: x.fillna(daily_avg))<jupyter_output><empty_output><jupyter_text>Fit PCA<jupyter_code>cov = np.cov(returns, rowvar=False) 
pca = PCA(n_components='mle')
pca.fit(returns)<jupyter_output><empty_output><jupyter_text>Visualize Explained Variance<jupyter_code>fig, axes = plt.subplots(ncols=2, figsize=(14,4))
pd.Series(pca.explained_variance_ratio_).iloc[:15].sort_values().plot.barh(title='Explained Variance Ratio by Top Factors',ax=axes[0]);
pd.Series(pca.explained_variance_ratio_).cumsum().plot(ylim=(0,1),ax=axes[1], title='Cumulative Explained Variance');
risk_factors = pd.DataFrame(pca.transform(returns)[:, :2], 
                            columns=['Principal Component 1', 'Principal Component 2'], 
                            index=returns.index)
risk_factors.info()
risk_factors['Principal Component 1'].corr(risk_factors['Principal Component 2'])
risk_factors.plot(subplots=True, figsize=(14,6));<jupyter_output><empty_output><jupyter_text>Simulation for larger number of stocks<jupyter_code>idx = pd.IndexSlice
with pd.HDFStore('../../data/assets.h5') as store:
    returns = (store['quandl/wiki/prices']
              .loc[idx['2000': '2018', :], 'adj_close']
              .unstack('ticker')
              .pct_change())
pca = PCA()
n_trials, n_samples = 100, 500
explained = np.empty(shape=(n_trials, n_samples))
for trial in range(n_trials):
    returns_sample = returns.sample(n=n_samples)
    returns_sample = returns_sample.dropna(thresh=int(returns_sample.shape[0] * .95), axis=1)
    returns_sample = returns_sample.dropna(thresh=int(returns_sample.shape[1] * .95))
    daily_avg = returns_sample.mean(1)
    returns_sample = returns_sample.apply(lambda x: x.fillna(daily_avg))
    pca.fit(returns_sample)
    explained[trial, :len(pca.components_)] = pca.explained_variance_ratio_
explained = pd.DataFrame(explained, columns=list(range(1, explained.shape[1] + 1)))
explained.info()
fig, axes =plt.subplots(ncols=2, figsize=(14, 4.5))
pc10 = explained.iloc[:, :10].stack().reset_index()
pc10.columns = ['Trial','Principal Component', 'Value']

pc10['Cumulative'] = pc10.groupby('Trial').Value.transform(np.cumsum)
sns.barplot(x='Principal Component', y='Value', data=pc10, ax=axes[0])
sns.lineplot(x='Principal Component', y='Cumulative', data=pc10, ax=axes[1])
fig.suptitle('Explained Variance of Top 10 Principal Components | 100 Trials')
fig.tight_layout()
fig.subplots_adjust(top=.90);<jupyter_output><empty_output><jupyter_text>Eigenportfolios<jupyter_code>idx = pd.IndexSlice
with pd.HDFStore('../../data/assets.h5') as store:
    stocks = store['us_equities/stocks'].marketcap.nlargest(30)
    returns = (store['quandl/wiki/prices']
               .loc[idx['2010': '2018', stocks.index], 'adj_close']
               .unstack('ticker')
               .pct_change())
normed_returns = scale(returns
                       .clip(lower=returns.quantile(q=.025), 
                             upper=returns.quantile(q=.975), 
                             axis=1)
                      .apply(lambda x: x.sub(x.mean()).div(x.std())))
returns = returns.dropna(thresh=int(returns.shape[0] * .95), axis=1)
returns = returns.dropna(thresh=int(returns.shape[1] * .95))
returns.info()
cov = returns.cov()
sns.clustermap(cov);
pca = PCA()
pca.fit(cov)
pd.Series(pca.explained_variance_ratio_).to_frame('Explained Variance').head().style.format('{:,.2%}'.format)
top4 = pd.DataFrame(pca.components_[:4], columns=cov.columns)
eigen_portfolios = top4.div(top4.sum(1), axis=0)
eigen_portfolios.index = [f'Portfolio {i}' for i in range(1, 5)]
eigen_portfolios.T.plot.bar(subplots=True, layout=(2,2), figsize=(14,6), legend=False, sharey=True);
fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(14,8), sharex=True)
axes = axes.flatten()
returns.mean(1).add(1).cumprod().sub(1).plot(title='The Market', ax=axes[0]);
for i in range(3):
    returns.mul(eigen_portfolios.iloc[i]).sum(1).add(1).cumprod().sub(1).plot(title=f'Portfolio {i+1}', ax=axes[i+1]);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16715947171299034, 'fraction_numerical': 0.036073329390892965, 'mean_word_length': 4.681970884658455, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '175342', 'license': 'MIT', 'n_tokens_mistral': 1968, 'n_tokens_neox': 1813, 'n_words': 268, 'path': 'Chapter12/01_linear_dimensionality_reduction/03_pca_and_risk_factor_models.ipynb', 'repo_name': 'PacktPublishing/Hands-On-Machine-Learning-for-Trading', 'size': '5073'}