<jupyter_start><jupyter_text>init<jupyter_code>from google.colab import drive
drive.mount('/gdrive')

%cd /gdrive/My Drive/root ap/Epoch and Batches
import sys
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
import random
import numpy as np
import torch
from pprint import pprint as pprint<jupyter_output><empty_output><jupyter_text>EnB<jupyter_code>from EpochAndBatches import EpochAndBatches as EnB<jupyter_output><empty_output><jupyter_text>unit test<jupyter_code>config = {}
config['n_epoch'] = 20
config['batch_size'] = 4096
config['n_batch'] = 30
config['device'] = 'cuda:0'
d = config['device']
enb = EnB(config['n_epoch'],config['n_batch'])
enb.config = config
# enb.verbose = 9000

enb.save()

pprint(vars(enb))
net = torch.nn.Sequential(torch.nn.Linear(1, 4096),
                          torch.nn.ReLU(),
                          torch.nn.Linear(4096, 1)).to(d)

o = torch.optim.Adam(net.parameters())

loss_mae = torch.nn.L1Loss()
loss_mse = torch.nn.MSELoss()<jupyter_output><empty_output><jupyter_text>fit<jupyter_code>enb.best_epoch = -1
enb.best_val_loss = sys.float_info.max
for i_epoch in range(config['n_epoch']):
    enb.next_epoch()
    z_mae = 0
    z_mse = 0
    zmetric1 = 0
    zmetric2 = 0
    # train one epoch
    for i_batch in range(config['n_batch']):
        enb.next_batch()
        # forward
        net.train()
        o.zero_grad()
        ix = torch.rand(config['batch_size'],1).to(d) * 4 - 2
        iy = ix**2
        ih = net(ix)
        mae = loss_mae(ih,iy)
        mse = loss_mse(ih,iy)
        loss = mae**2 + mse
        # backward
        loss.backward()
        o.step()
        # metrics
        metric1 = mae.item() + mse.item()
        metric2 = mae.item() - mse.item()
        z_mae += mae.item()
        z_mse += mse.item()
        zmetric1 += metric1
        zmetric2 += metric2
        #enb
        enb.log_batch('loss:mae',mae.item(),35)
        enb.log_batch('loss:mse',mse.item(),35)
        enb.log_batch('metric:1',metric1,35)
        enb.log_batch('metric:2',metric2,35)
    z_mae /= config['n_batch']
    z_mse /= config['n_batch']
    zmetric1 /= config['n_batch']
    zmetric2 /= config['n_batch']
    enb.log_epoch('loss:mae',z_mae,35)
    enb.log_epoch('loss:mse',z_mse,35)
    enb.log_epoch('metric:1',zmetric1,35)
    enb.log_epoch('metric:2',zmetric2,35)
    enb.log_epoch('gpu-mem',str(torch.cuda.memory_reserved(d)),35)
    # validate one epoch
    net.eval()
    ix = torch.rand(config['batch_size']*5,1).to(d) *6 - 3
    iy = ix**2
    ih = net(ix)
    val_mae = loss_mae(ih,iy).item()
    val_mse = loss_mse(ih,iy).item()
    loss = val_mae**2 + val_mse
    val_metric1 = val_mae + val_mse
    val_metric2 = val_mae - val_mse
    enb.log_epoch('loss:val_mae',val_mae,35)
    enb.log_epoch('loss:val_mse',val_mse,35)
    enb.log_epoch('metric:1',val_metric1,35)
    enb.log_epoch('metric:2',val_metric2,35)
    enb.log_epoch('model:net_state_dict',net.state_dict(),35)
    enb.log_epoch('optim:adam_state_dict',o.state_dict(),35)
    if loss < enb.best_val_loss:
        enb.best_val_loss = loss
        enb.best_epoch = enb.i_epoch
        enb.best_model_state_dict = net.state_dict()
        enb.log_msg('New best_val_loss='+str(enb.best_val_loss))
        enb.best_val_mae = val_mae
        enb.best_val_mse = val_mse
        enb.best_val_metric1 = val_metric1
        enb.best_val_metric2 = val_metric2
#test
net.load_state_dict(enb.best_model_state_dict)
ix = torch.rand(config['batch_size']*5,1).to(d) * 8 - 4
iy = ix**2
ih = net(ix)
enb.tst_mae = loss_mae(ih,iy).item()
enb.tst_mse = loss_mse(ih,iy).item()
enb.tst_metric1 = enb.tst_mae + enb.tst_mse
enb.tst_metric2 = enb.tst_mae - enb.tst_mse
enb.save()<jupyter_output>EnB | msg = next_epoch: 0/20  |  00:00:00 + 00:00:00 = 00:00:00 / 00:00:00
EnB | msg = New best_val_loss=0.6156449827258257
EnB | msg = next_epoch: 1/20  |  00:00:01 + 00:00:20 = 00:00:01 / 00:00:21
EnB | msg = New best_val_loss=0.4570068904295903
EnB | msg = next_epoch: 2/20  |  00:00:02 + 00:00:19 = 00:00:01 / 00:00:21
EnB | msg = New best_val_loss=0.3574326042167053
EnB | msg = next_epoch: 3/20  |  00:00:03 + 00:00:18 = 00:00:01 / 00:00:21
EnB | msg = New best_val_loss=0.2825830862922827
EnB | msg = next_epoch: 4/20  |  00:00:04 + 00:00:17 = 00:00:01 / 00:00:21
EnB | msg = New best_val_loss=0.24570834631221516
EnB | msg = next_epoch: 5/20  |  00:00:05 + 00:00:16 = 00:00:01 / 00:00:21
EnB | msg = New best_val_loss=0.23792475520781609
EnB | msg = next_epoch: 6/20  |  00:00:06 + 00:00:15 = 00:00:01 / 00:00:21
EnB | msg = New best_val_loss=0.2184428196013235
EnB | msg = next_epoch: 7/20  |  00:00:07 + 00:00:14 = 00:00:01 / 00:00:21
EnB | msg = New best_val_loss=0.20539241449972523
EnB |[...]<jupyter_text>load<jupyter_code>enb2 = EnB.load(enb.enb_output_filepath)
d_enb = vars(enb2)
for k,v in d_enb.items():
    print(k,type(v))<jupyter_output>project_name <class 'str'>
sweep_name <class 'str'>
dflog <class 'pandas.core.frame.DataFrame'>
script_start_time <class 'float'>
pid <class 'int'>
cwd <class 'str'>
verbose <class 'int'>
n_epoch <class 'int'>
n_batch <class 'int'>
i_epoch <class 'int'>
i_batch <class 'int'>
is_save_per_log <class 'bool'>
is_save_per_batch <class 'bool'>
is_save_per_epoch <class 'bool'>
run_name <class 'str'>
enb_output_filepath <class 'str'>
config <class 'dict'>
best_epoch <class 'int'>
best_val_loss <class 'float'>
fit_start_time <class 'float'>
i_batch_start_time <class 'float'>
best_model_state_dict <class 'collections.OrderedDict'>
best_val_mae <class 'float'>
best_val_mse <class 'float'>
best_val_metric1 <class 'float'>
best_val_metric2 <class 'float'>
tst_mae <class 'float'>
tst_mse <class 'float'>
tst_metric1 <class 'float'>
tst_metric2 <class 'float'><jupyter_text>display<jupyter_code>for il in enb2.dflog['label'].unique():
    for ik in enb2.dflog['key'].loc[enb2.dflog['label']==il].unique():
        print(il,ik)
for il in enb2.dflog['label'].unique():
    for ik in enb2.dflog['key'].loc[enb2.dflog['label']==il].unique():
        print(il,ik)
        display(enb2.dflog.loc[(enb2.dflog['key']==ik) & (enb2.dflog['label']==il)])
i_label = 'per_epoch'
i_key_group = 'loss'

plt.figure(figsize=[20,10])
plt.xlabel('i_epoch')
plt.ylabel('MinMax scaled values')
for ik in enb2.dflog['key'].loc[enb2.dflog['label']==i_label].unique():
    if ik[:len(i_key_group)] != i_key_group: continue
    tdf = enb2.dflog.loc[(enb2.dflog['key']==ik) & (enb2.dflog['label']==i_label)]
    plt.semilogy(tdf['i_epoch'],(tdf['value']-tdf['value'].min())/(tdf['value'].max()-tdf['value'].min()),label=ik)
plt.legend(framealpha=0.7)<jupyter_output><empty_output><jupyter_text>scratch<jupyter_code>spam = torch.nn.Parameter(torch.randn(1).to(d))
print('',spam)
print('.detach()',spam.detach())
print('.cpu()',spam.cpu())
print('.item()',spam.item())
print('.detach().cpu()',spam.detach().cpu())
print('.detach().cpu().item()',spam.detach().cpu().item())<jupyter_output>Parameter containing:
tensor([-1.9656], device='cuda:0', requires_grad=True)
.detach() tensor([-1.9656], device='cuda:0')
.cpu() tensor([-1.9656], grad_fn=<CopyBackwards>)
.item() -1.9656450748443604
.detach().cpu() tensor([-1.9656])
.detach().cpu().item() -1.9656450748443604
{'fraction_non_alphanumeric': 0.1720266149154422, 'fraction_numerical': 0.0752703077349598, 'mean_word_length': 4.27027027027027, 'pattern_counts': {'":': 0, '<': 58, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '254192', 'license': 'MIT', 'n_tokens_mistral': 3378, 'n_tokens_neox': 3114, 'n_words': 480, 'path': 'eandb_test.ipynb', 'repo_name': 'aprbw/EpochAndBatches', 'size': '7214'}