<jupyter_start><jupyter_text>In this notebook:* Static training and evaluation* Rolling training and evaluation* Trade activity plot and performance comparison* Explainability plots: input gradients and UMAP<jupyter_code>SAVE_PATH = Path('../plots2_0').resolve()
CURR_IDX = 0
LABEL_START_IDX = 1123
CURRENCIES = ['USDCNY', 'USDIDR', 'USDINR', 'USDKRW', 'USDPHP', 'USDTWD']
CURR_N = CURRENCIES[CURR_IDX]
device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')<jupyter_output><empty_output><jupyter_text>Data prep:<jupyter_code>PATH = Path('../../data/final').resolve()
input_data = pd.read_pickle(f'{PATH}/input_data_labels')
returns = pd.read_pickle(f'{PATH}/returns')
expert_oracle_labels = pd.read_pickle(f'{PATH}/expert_oracle_labels')
data, expert_labels = input_data.iloc[:, :LABEL_START_IDX], input_data.iloc[:, LABEL_START_IDX:]
dh = FXDataHandler()
dh.assign_data(data)
dh.split_train_val_test([60, 30, 10])
dh.to_tensor(device=device)
train, val, test = dh.datasets

len_slice = 30
bs = 32

trainloader = DataLoader(RegularFinancialData(train, len_slice),
                         batch_size=bs, shuffle=True)

# differently from the recurrent models, we batch validation sequences
# since WATTNet attention blocks are memory intensive 
valloader = DataLoader(RegularFinancialData(val, len_slice), 
                       batch_size=len(val), shuffle=False)
# split returns into train, val, test returns
len(train), len(val), len(test)
# returns from last day for sequence of length `len_slice`
train_returns = returns[:, len_slice:len(train)+len_slice, :]

val_returns = returns[:, len_slice+len(train): \
                 len(train)+len(val), :]

test_returns = returns[:, len_slice+len(train)+len(val): \
                  len(train)+len(val)+len(test), :]<jupyter_output><empty_output><jupyter_text>Static Validation<jupyter_code>train_returns_single = torch.Tensor(train_returns[:, :, CURR_IDX]).to(device)
val_returns_single = torch.Tensor(val_returns[:, :, CURR_IDX]).to(device)

ensemble_size=1
ensemble = []

# seeds
seed_list = []
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False

for _ in range(ensemble_size):
    torch.manual_seed(_)
    np.random.seed(_)
    torch.cuda.manual_seed_all(_)
    seed_list.append((_, _, _))
    m = WATTNet(in_dim=1123, w_dim=90, emb_dim=1, dilation_depth=2, dropout_prob=0.2, n_repeat=4).to(device)
    opt = torch.optim.Adam(m.parameters(), lr=0.006, weight_decay=0.0003)
    sched = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(opt, 200, eta_min=0.003, T_mult=2)
    m.fit(epochs=200, 
          trainloader=trainloader, 
          valloader=valloader, 
          opt=opt, 
          sched=sched, 
          device=device, 
          log_interval=1000, 
          dropout_schedule=[0.2, 0.2, 0., 0.], 
          dropout_interval=50,
          tr_returns=train_returns_single,
          val_returns=val_returns_single
          )
    
    ensemble.append(m)<jupyter_output>Epoch: 0
Training Loss: 4.77
Avg train returns: 0.25
Avg train optimal returns: 0.63 

Validation Loss: 7.61
Validation Accuracy: 8.17 %
Avg val returns: 0.0
Avg val optimal returns: 7.6 

Epoch: 1
Training Loss: 4.03
Avg train returns: 0.21
Avg train optimal returns: 0.49 

Validation Loss: 4.15
Validation Accuracy: 7.93 %
Avg val returns: -0.31
Avg val optimal returns: 7.6 

Epoch: 2
Training Loss: 3.84
Avg train returns: 0.21
Avg train optimal returns: 0.57 

Validation Loss: 4.11
Validation Accuracy: 8.65 %
Avg val returns: 0.78
Avg val optimal returns: 7.6 

Epoch: 3
Training Loss: 3.67
Avg train returns: 0.15
Avg train optimal returns: 0.39 

Validation Loss: 4.25
Validation Accuracy: 8.65 %
Avg val returns: -0.18
Avg val optimal returns: 7.6 

Epoch: 4
Training Loss: 3.67
Avg train returns: 0.19
Avg train optimal returns: 0.42 

Validation Loss: 4.31
Validation Accuracy: 7.21 %
Avg val returns: 0.58
Avg val optimal returns: 7.6 

Epoch: 5
Training Loss: 3.79
Avg train returns: 0[...]<jupyter_text>Rolling Validation<jupyter_code>static_train_l = len(train)
static_val_l = len(val)
PATH = Path('../../data/final').resolve()
input_data = pd.read_pickle(f'{PATH}/input_data_labels')
returns = pd.read_pickle(f'{PATH}/returns')
expert_oracle_labels = pd.read_pickle(f'{PATH}/expert_oracle_labels')



len_slice = 30
bs = 32
chunk_size = 800
stride_length = 1
val_chunk_size = 50

# use static validation training and val set dimensions to calculate
# start and end day index of rolling validation procedure
# NOTE: final `+ len_slice` is required to match static validation number of 
START_IDX = static_train_l - (chunk_size + 91) + len_slice  

# trading days 
END_IDX = static_train_l + static_val_l - (chunk_size + 91)   
n_rolling_runs = (END_IDX - START_IDX)//(stride_length*val_chunk_size)

# split labels from data 
data, expert_labels = input_data.iloc[:, :LABEL_START_IDX], input_data.iloc[:, LABEL_START_IDX:]
# to store ROI and acc
rets, mod_acc, mod_oacc, retse, retseo, retsop = [], [], [], [], [], []
jump = stride_length + val_chunk_size

for run in range(1):
    for j in range(n_rolling_runs):
        # data for specific rolling run: chunk of 360 + 91 days
        # 360 are used for training and the single point at 360 + 91 for out-of-sample
        # validation
        run_data = data[j*stride_length : START_IDX + j*jump + chunk_size + 91 + val_chunk_size]
        run_returns = torch.Tensor(returns[:, START_IDX + j*jump : \
                                           START_IDX + j*jump + chunk_size + 91 + val_chunk_size, CURR_IDX]).to(device)

        dh = FXDataHandler()
        dh.assign_data(run_data)
        # split done manually 
        dh.split_train_val_test([100, 0, 0])
        dh.to_tensor(device=device)
        tensor_data, _, _ = dh.datasets

        train = tensor_data[:chunk_size]
        # val contains a single slice of len `len_slice` which ends
        # with trading day 360 + 91
        val = tensor_data[-len_slice:]
        train_returns = run_returns[:, len_slice:len(train)+len_slice]
        # 0-indexing: 91 -> 90
        val_returns = run_returns[:, len(train)+ 90 : len(train)+ 90 + val_chunk_size]

        expo = expert_oracle_labels[CURR_IDX][START_IDX + j*jump + chunk_size + 90 : \
                                              j*stride_length + chunk_size + 90 + val_chunk_size]
        # exp from labels
        exp = expert_labels.values[START_IDX + j*jump + chunk_size + 90 : \
                                   j*stride_length + chunk_size + 90 + val_chunk_size, CURR_IDX]

        trainloader = DataLoader(RegularFinancialData(train, len_slice),
                                 batch_size=bs, shuffle=True)


        ensemble_size=1
        ensemble = []
        # seeds
        seed_list = []
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False
        
        print(f'***** Processing slice: {j} ***** \n')
        
        for _ in range(ensemble_size):
            torch.manual_seed(_)
            np.random.seed(_)
            torch.cuda.manual_seed_all(_)
            seed_list.append((_, _, _))

            m = WATTNet(w_dim=12, emb_dim=1, dilation_depth=2, dropout_prob=0.2, n_repeat=4).to(device)
            opt = torch.optim.Adam(m.parameters(), lr=0.006, weight_decay=0.0003)
            sched = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(opt, 200, eta_min=0.003, T_mult=2)
            m.fit(epochs=50, 
                  trainloader=trainloader, 
                  valloader=None, 
                  opt=opt, 
                  sched=sched, 
                  device=device, 
                  log_interval=1000, 
                  dropout_schedule=[0.2, 0.2, 0., 0.], 
                  dropout_interval=50,
                  tr_returns=train_returns,
                  val_returns=None
                  )
            m.eval()
            m.postMLP.drop_probability = 0.
            ensemble.append(m)

        

        # make batch of size 1 with unsqueeze
        val = val.unsqueeze(0)
        val = val[:, :-1]
        _, optimal_act = val_returns.max(0)
        for i in range(ensemble_size):
            prediction = ensemble_predictions(ensemble=[ensemble[i]], data=val, device=device)
            retmod = val_returns[prediction]

            acc_opt = (prediction.to(device) == optimal_act)
            acc_pos = retmod >= 0
            mod_oacc.append(acc_opt)
            mod_acc.append(acc_pos)
            
            retexp = val_returns[exp]
            retexpo = val_returns[expo]
            retsopt, _ = val_returns.max(0)
            rets.append(retmod.sum().item())
            retse.append(retexp.sum().item())
            retseo.append(retexpo.sum().item())
            retsop.append(retsopt.sum().item())
            
            print(sum(rets), sum(retsop))<jupyter_output><empty_output><jupyter_text>Actions visualization and performance metrics<jupyter_code># remove dropout
for model in ensemble:
    model.eval()
    model.postMLP.drop_probability = 0.
    
# name from input_data columns

curr_n = CURRENCIES[CURR_IDX]
# exp oracle from separate oracle np.array
expo = expert_oracle_labels[CURR_IDX][len(train)+len_slice:len(train)+len(val)]
# exp from labels
exp = expert_labels.values[len(train)+len_slice:len(train)+len(val), CURR_IDX]
# validation returns for single currency, first `len_slice` days are removed to match predictions
valr = val_returns[:, :, CURR_IDX]

all_batch_predictions = []
all_val_data = []
iterator = iter(valloader)
for it in range(len(valloader)):
    # append all batches for metric computation later
    data, _ = next(iterator)
    all_val_data.append(data)
    
    # WATTNet takes 4 dim. input
    data = data[:, :-1, :].unsqueeze(1)
    predictions = ensemble_predictions(ensemble=[ensemble[0]], data=data, device='cuda')
    all_batch_predictions.append(predictions)
                                       
all_batch_predictions = torch.cat(all_batch_predictions)
all_val_data = torch.cat(all_val_data)

plot_activity(all_batch_predictions, model_name='WATTNet', curr_name=curr_n, expert_labels=expo, returns=valr)
#plt.savefig(f'{SAVE_PATH}/WATT_{CURR_N}_ACTIVITY.jpg', dpi=200)<jupyter_output><empty_output><jupyter_text>UMAP of latents<jupyter_code># hook the latents out of the first post conv MLP layer
model = ensemble[0].to(device)

latents = []
pred_ensemble = []
iterator = iter(valloader)

data, _ = next(iterator)
data_clip = data[:,:-1,:].unsqueeze(1)
h = Hook(model._modules['postMLP'])  
yhat = model(data_clip)
probs = nn.Softmax(1)(yhat)  
_, predictions = torch.max(probs, 1)
latents = h.input[0]
# compute action indices for different baselines
opt_idxs = []
exp_idxs = []
mod_idxs = []
opt_actions = valr.argmax(0)
for i in range(91):
    opt_idxs.append((opt_actions==i).nonzero()[0])
    exp_idxs.append((expo==i).nonzero()[0])
    mod_idxs.append((predictions==i).nonzero().cpu())
for i in range(91): 
    opt_idxs[i] = np.array(opt_idxs[i].flatten())
    exp_idxs[i] = np.array(exp_idxs[i].flatten())
    mod_idxs[i] = np.array(mod_idxs[i].flatten())

pca = UMAP(n_components=3)
emb = pca.fit_transform(latents.detach().cpu().numpy())
plot_trade_mode_embeddings(emb, mod_idxs, dims_to_plot=2)
#plt.savefig(f'{curr_n}, trade_behavior_PCA.jpg', dpi=200)<jupyter_output><empty_output><jupyter_text>Effectiveness: ROI and additional metrics  Returns for all baselines<jupyter_code>data = all_val_data
valr = torch.Tensor(valr)
# expert actions
texp = torch.Tensor(exp).to(dtype=torch.long)
# expert oracle
texpo = torch.Tensor(expo).to(dtype=torch.long)
# optimal
_, topt = valr.max(0)
topt = topt.to(dtype=torch.long)
# last_seen (Momentum)
ls_exp = texp
ls_exp[1:] = ls_exp[0:-1]
ls_exp[0] = 0


modelContainer = returns_and_activity(returns=valr, predictions=all_batch_predictions, baseline=None, input_data=data)
expertContainer = returns_and_activity(returns=valr, predictions=texpo, baseline='expert', input_data=data)
oracleContainer = returns_and_activity(returns=valr, predictions=texpo, baseline='expert', input_data=data)
optimalContainer = returns_and_activity(returns=valr, predictions=topt, baseline='expert', input_data=data)
momentumContainer = returns_and_activity(returns=valr, predictions=ls_exp, baseline='expert', input_data=data)
randomContainer = returns_and_activity(returns=valr, predictions=None, baseline='random', input_data=data)
modelContainer<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11927710843373494, 'fraction_numerical': 0.02465863453815261, 'mean_word_length': 3.4707360861759424, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '597145', 'license': 'MIT', 'n_tokens_mistral': 4556, 'n_tokens_neox': 4230, 'n_words': 870, 'path': 'notebooks/compact/06_wattnet_results (compact).ipynb', 'repo_name': 'Zymrael/wattnet-fx-trading', 'size': '12450'}