<jupyter_start><jupyter_text>Connect to Google Drive<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
import gc
import numpy as np
import pandas as pd

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from matplotlib import pyplot as plt

import random as python_random

import warnings
warnings.simplefilter('ignore')
np.random.seed(1)
python_random.seed(1)
tf.random.set_seed(1)
# CONSTANTS:
TOURNAMENT_NAME = ""
TARGET_NAME = f"target{TOURNAMENT_NAME}"
PREDICTION_NAME = f"prediction{TOURNAMENT_NAME}"<jupyter_output><empty_output><jupyter_text>Read training data<jupyter_code>training_data = pd.read_csv("https://numerai-public-datasets.s3-us-west-2.amazonaws.com/latest_numerai_training_data.csv.xz")
training_data<jupyter_output><empty_output><jupyter_text>Define function for plotting loss curve<jupyter_code>def plot_the_loss_curve(epochs, mse):
    """Plot a curve of loss vs. epoch."""
    plt.figure()
    plt.xlabel("Epoch")
    plt.ylabel("Mean Squared Error")

    plt.plot(epochs, mse, label="Loss")
    plt.legend()
    plt.ylim([mse.min() * 0.95, mse.max() * 1.03])
    plt.draw()<jupyter_output><empty_output><jupyter_text>Define functions for validation<jupyter_code>def ar1(x):
    return np.corrcoef(x[:-1], x[1:])[0,1]

def autocorr_penalty(x):
    n = len(x)
    p = ar1(x)
    return np.sqrt(1 + 2*np.sum([((n - i)/n)*p**i for i in range(1,n)]))

def sharpe(x):
    return np.mean(x)/np.std(x, ddof=1)

def smart_sharpe(x):
    return np.mean(x)/(np.std(x, ddof=1)*autocorr_penalty(x))

def numerai_sharpe(x):
    return ((np.mean(x) - 0.010415154) / np.std(x)) * np.sqrt(12)

def spearmanr(target, pred):
    return np.corrcoef(
        target,
        pred.rank(pct=True, method="first")
    )[0, 1]

def sortino_ratio(x, target=0.010415154):
   xt = x# - target
   return np.mean(xt) / (np.sum(np.minimum(0, xt)**2)/(len(xt)-1))**.5

def score(df):
    # method="first" breaks ties based on order in array
    pct_ranks = df[PREDICTION_NAME].rank(pct=True, method="first")
    targets = df[TARGET_NAME]
    return np.corrcoef(targets, pct_ranks)[0, 1]<jupyter_output><empty_output><jupyter_text>Build model<jupyter_code>def build_model(learning_rate, layer_size1, layer_size2):
    model = keras.Sequential([
      layers.Dense(layer_size1, activation='relu',input_shape=[len(feature_names)]),
      layers.Dense(layer_size2, activation='relu'
                   , kernel_regularizer=tf.keras.regularizers.l2(0.01)
                  ),
      layers.Dense(1, activation=('sigmoid'))
                   ])
    optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)

    model.compile(loss='mse', optimizer=optimizer, metrics=['mae', 'mse'])
    
    return model<jupyter_output><empty_output><jupyter_text>Define training model function<jupyter_code>def train_model(model, feature, label, epochs, batch_size):
    # tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir="log_dir", histogram_freq=1)
    # early_stopping_callback = tf.keras.callbacks.EalryStopping(monitor='val_loss', min_delta=0, patience=0, verbose=0, mode='auto',
    # baseline=None, restore_best_weights=False)
    history = model.fit(x=feature,
                        y=label,
                        batch_size=batch_size,
                        epochs=epochs,
                        verbose=0,
                        # callbacks=[tensorboard_callback]
                       )
    epochs = history.epoch
    hist = pd.DataFrame(history.history)
    mse = hist["mse"]
    
    return epochs, mse
# Define features
feature_names = [
    f for f in training_data.columns if f.startswith("feature")]

# Set features and target for training
train_features = training_data[feature_names]
train_target = training_data[TARGET_NAME]

# Set Parameters:
lr = 0.0001
# epochs = 10
batch_size = 1024
layer_Size1 = 32
layer_Size2 = 32<jupyter_output><empty_output><jupyter_text>Build model<jupyter_code>model = build_model(learning_rate=lr, layer_size1=layer_Size1, layer_size2=layer_Size2)
model.summary()<jupyter_output>Model: "sequential_1"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 dense_3 (Dense)             (None, 32)                9952      
                                                                 
 dense_4 (Dense)             (None, 32)                1056      
                                                                 
 dense_5 (Dense)             (None, 1)                 33        
                                                                 
=================================================================
Total params: 11,041
Trainable params: 11,041
Non-trainable params: 0
_________________________________________________________________<jupyter_text>train<jupyter_code>import matplotlib.pyplot as plt

def era_boost_train(X, y, era_col, proportion=0.5, epochs_per_step=10, num_iters=200):
    n_epochs = epochs_per_step
    features = X.columns
    train_model(model, X, y, n_epochs, batch_size)
    
    new_df = X.copy()
    new_df["target"] = y
    new_df["era"] = era_col

    for i in range(num_iters-1):
        print("---------------")
        print(f"iteration {i}")

        # score each era
        preds = model.predict(X)
        new_df["pred"] = preds
        era_scores = pd.Series(index=new_df["era"].unique())
        
        for era in new_df["era"].unique():
            era_df = new_df[new_df["era"] == era]
            # print(era_df.columns)
            era_scores[era] = spearmanr(era_df["pred"], era_df["target"])

        era_scores.sort_values(inplace=True)
        worst_eras = era_scores[era_scores <= era_scores.quantile(proportion)].index
        print("worst eras: ", list(worst_eras))
        worst_df = new_df[new_df["era"].isin(worst_eras)]
        era_scores.sort_index(inplace=True)
        era_scores.plot(kind="bar")

        print("")
        print("performance over time")
        plt.show()
        print("")

        # show performance
        print("autocorrelation: ", ar1(era_scores))
        print("mean correlation: ", np.mean(era_scores))
        print("sharpe: ", np.mean(era_scores)/np.std(era_scores))
        print("smart sharpe: ", smart_sharpe(era_scores))

        print(smart_sharpe(era_scores))
        n_epochs += epochs_per_step
        train_model(model, worst_df[features], worst_df["target"], n_epochs, batch_size)

    return model

boost_model = era_boost_train(train_features, train_target, era_col=training_data["era"], proportion=0.5, epochs_per_step=1, num_iters=20)
# model.save('...h5')<jupyter_output><empty_output><jupyter_text>Read tournament data<jupyter_code>tournament_data = pd.read_csv("https://numerai-public-datasets.s3-us-west-2.amazonaws.com/latest_numerai_tournament_data.csv.xz")
tournament_data<jupyter_output><empty_output><jupyter_text>Set validation data<jupyter_code>validation_data = tournament_data[tournament_data['data_type'] == 'validation']
validation_data['era']
del tournament_data
gc.collect()<jupyter_output><empty_output><jupyter_text>Predict<jupyter_code>predictions = model.predict(validation_data[feature_names])
validation_data[PREDICTION_NAME] = predictions
validation_correlations = validation_data.groupby("era").apply(score)
print(f"On validation the correlation has mean {validation_correlations.mean()} and std {validation_correlations.std()}")
print("sharpe: ", sharpe(validation_correlations))
print("smart sharpe: ", smart_sharpe(validation_correlations))
print("sortino_ratio: ", sortino_ratio(validation_correlations))<jupyter_output>sharpe:  0.6590148561908438
smart sharpe:  0.6554492576482689
sortino_ratio:  1.4785723873297398<jupyter_text>Neutralize<jupyter_code>from sklearn.preprocessing import MinMaxScaler
def _neutralize(df, columns, by, proportion=1.0):
    scores = df[columns]
    exposures = df[by].values
    scores = scores - proportion * exposures.dot(np.linalg.pinv(exposures).dot(scores))
    return scores / scores.std()
def _normalize(df):
    X = (df.rank(method="first") - 0.5) / len(df)
    return scipy.stats.norm.ppf(X)
def normalize_and_neutralize(df, columns, by, proportion=1.0):
    # Convert the scores to a normal distribution
    df[columns] = _normalize(df[columns])
    df[columns] = _neutralize(df, columns, by, proportion)
    return df[columns]
import scipy
validation_data["preds"] = predictions
validation_data["preds_neutralized"] = validation_data.groupby("era").apply(
    lambda x: normalize_and_neutralize(x, ["preds"], feature_names, 0.5) # neutralize by 50% within each era
)
scaler = MinMaxScaler()
validation_data["preds_neutralized"] = scaler.fit_transform(validation_data[["preds_neutralized"]]) # transform back to 0-1
validation_data[PREDICTION_NAME] = validation_data['preds_neutralized']
validation_correlations = validation_data.groupby("era").apply(score)
print(f"On validation the correlation has mean {validation_correlations.mean()} and std {validation_correlations.std()}")
print("sharpe: ", sharpe(validation_correlations))
print("smart sharpe: ", smart_sharpe(validation_correlations))
print("sortino_ratio: ", sortino_ratio(validation_correlations))<jupyter_output>sharpe:  0.6590148561908438
smart sharpe:  0.6554492576482689
sortino_ratio:  1.4785723873297398
{'fraction_non_alphanumeric': 0.13583262531860663, 'fraction_numerical': 0.026338147833474938, 'mean_word_length': 3.948502364687336, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 46, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '560836', 'license': 'MIT', 'n_tokens_mistral': 3305, 'n_tokens_neox': 3089, 'n_words': 525, 'path': 'train_tensorflow_densenet_eraboost_2021_12_24.ipynb', 'repo_name': 'mst-projects/numerai', 'size': '9416'}