<jupyter_start><jupyter_text>Principle Component Explained Variance <NAME><jupyter_code># Standard imports 
import numpy as np
import matplotlib.pyplot as plt
plt.style.use('ggplot')
from subsets import *
from coin_helpers import load_coins, simplify

coins, returns = load_coins('pairs/', top_50)
hour_ret = simplify(returns, '1h')
day_ret = simplify(returns, '1D')
n = 365
twoyr, oneyr = [], []
for c in day_ret.columns:
    nulls = day_ret[c].isnull().sum()
    lengs = day_ret[c].shape[0]
    if lengs - nulls > n:
        oneyr.append(c)
        if lengs - nulls > 2*n:
            twoyr.append(c)

nyr = len(oneyr)
nall = len(returns.columns)
p = round(100 * nyr / nall, 2)
print("{0} of {1} ({2}%) have more than {3} days of data".format(nyr, nall, p, n))
print("{0} of {1} ({2}%) have more than {3} days of data".format(len(twoyr), nall, p, 2 * n))
# Grab historical subset and drop bad days
twoall = returns[twoyr].count(axis=1) 
tpca = returns[twoyr].dropna(how='any').count(axis=1)
oneall = returns[oneyr].count(axis=1)
opca = returns[oneyr].dropna(how='any').count(axis=1)

# Illustrate relationship
fig = plt.figure(figsize=(10,6))
plt.plot(oneall, label='Raw One Year', lw=2.5)
plt.plot(opca, label='Filtered One Year', lw='3')
plt.plot(twoall, label='Raw Two Year', ls='--')
plt.plot(tpca, label='Filtered Two Year', ls='--', lw='3')
plt.title("Num Coins vs Time")
plt.legend()
plt.show()
from sklearn.decomposition import PCA

# the Oxford paper suggests setting T s.t. T / N >= 1

def numpy_ma(x, w):
    return np.convolve(x, np.ones(w), 'valid') / w

def linear_pca_explained_var(ts, samps, T = 14, ma_window = 10):
    
    N = max(ts.dropna(how='any').count(axis=1))
    print(N)

    for samp in samps: 
        forpca = simplify(ts, samp).dropna(how='any')
        n_windows = forpca.shape[0] - T + 1
        ma_window = int(n_windows / 30)

        # print(n_windows, ma_window)
        
        pca = PCA(n_components=N)
        expl_vars = np.zeros(n_windows)
        windows = forpca.rolling(window=T)
        for i, window in enumerate(windows):
            if window.shape[0] < T: continue
            expl_vars[i - T] = pca.fit(window).explained_variance_ratio_[0]
        ma = numpy_ma(expl_vars, ma_window)

        fig = plt.figure(figsize=(12,8))
        plt.plot(forpca.index[-n_windows:], expl_vars,
                 label='Explained {}-Period Variance'.format(T))
        plt.plot(forpca.index[-ma.shape[0]:], ma, 
                 color='black', lw=2.5, label="{}-Period MA".format(ma_window))
        plt.title("{}-Interval LPCA Explained Variance for Oldest {} Coins".format(samp, len(forpca.columns)))
        plt.legend()
        plt.ylim([0, 1])
        plt.show()
linear_pca_explained_var(returns[twoyr], ['10min', '1h', '12h', '1D'], 14)
def sample_aggregated_linear_pca_explained_var(ts, samps, T = 14, ma_window = 10):
    
    N = max(ts.dropna(how='any').count(axis=1))
    print(N)

    fig = plt.figure(figsize=(12,8))

    for samp in samps: 
        forpca = simplify(ts, samp).dropna(how='any')
        n_windows = forpca.shape[0] - T + 1
        ma_window = int(n_windows / 30)

        # print(n_windows, ma_window)
        
        pca = PCA(n_components=N)
        expl_vars = np.zeros(n_windows)
        windows = forpca.rolling(window=T)
        for i, window in enumerate(windows):
            if window.shape[0] < T: continue
            expl_vars[i - T] = pca.fit(window).explained_variance_ratio_[0]
        ma = numpy_ma(expl_vars, ma_window)
        
        plt.plot(forpca.index[-ma.shape[0]:], ma, label="{}|{}-Period MA".format(samp, ma_window))
    
    plt.title("LPCA Explained Variance for Oldest {} Coins".format(len(forpca.columns)))
    plt.legend()
    plt.ylim([0, 1])
    plt.show()
sample_aggregated_linear_pca_explained_var(returns[twoyr], ['15min', '1h', '3h', '6h', '12h', '1D'], 14)<jupyter_output>14
{'fraction_non_alphanumeric': 0.16113256113256114, 'fraction_numerical': 0.023166023166023165, 'mean_word_length': 3.23773173391494, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '50897', 'license': 'MIT', 'n_tokens_mistral': 1529, 'n_tokens_neox': 1448, 'n_words': 312, 'path': 'notebooks/explained_variance.ipynb', 'repo_name': 'MattAHarrington/crypto-comovement', 'size': '3894'}