<jupyter_start><jupyter_text>Inputs<jupyter_code># batch, seq, n_out
x = torch.randn(2, 4, 5)
y = torch.randn(2, 4, 5)
x
y
m = torch.tensor([[0, 0, 1, 1], [0, 1, 1, 1]])
l = torch.tensor([[2], [3]])<jupyter_output><empty_output><jupyter_text>Loss by fancy<jupyter_code>criterion_fancy(x, y, m)<jupyter_output><empty_output><jupyter_text>Loss by loop<jupyter_code>loss = 0
for i in range(len(x)):
    x_clip = x[i,-l[i]:]
    y_clip = y[i,-l[i]:]
    print(x_clip)
    print(y_clip)
    loss_a = criterion(x_clip, y_clip)
    loss_b = criterion_red(x_clip, y_clip).sum() / 2
    print(loss_a)
    print(loss_b)
    loss += loss_a
    
    print('--')
print('==')
print(loss / len(x))<jupyter_output>tensor([[ 1.2433,  0.4055, -1.0698, -1.5804,  1.2499],
        [-1.9781, -1.3375, -1.2968, -0.8545,  0.8955]])
tensor([[ 1.0388, -0.3966, -0.2596,  1.9262, -0.1836],
        [-1.9481,  0.8885,  0.0687, -1.2673,  0.1870]])
tensor(0.4633)
tensor(2.3163)
--
tensor([[-0.1056, -1.4996,  0.2971,  1.0016, -0.2294],
        [-1.4290,  0.0747,  1.4130,  0.3132,  1.3201],
        [ 0.1978,  3.0526, -0.2047,  0.0086, -0.7705]])
tensor([[-0.4722,  1.8635,  0.5248,  0.6397,  0.9245],
        [ 2.1397,  0.0330,  1.1433,  0.0392, -0.6117],
        [ 0.9815,  1.5426, -1.9470, -0.3060,  0.6797]])
tensor(0.9187)
tensor(6.8900)
--
==
tensor(0.6910)<jupyter_text>Loss vectorized<jupyter_code># loss, but not reduce
loss_not_reduced = criterion_red(x, y)
loss_not_reduced
# apply mask
loss_masked_not_reduced = loss_not_reduced * m.unsqueeze(dim=-1)
loss_masked_not_reduced
# scale rows by num not masked
loss_masked_not_reduced_scalled = loss_masked_not_reduced / l.unsqueeze(dim=-1)
loss_masked_not_reduced_scalled
# quick test?
aa = loss_masked_not_reduced[0].sum() / l[0]
bb = loss_masked_not_reduced[1].sum() / l[1]
(aa + bb) / 2
# sum and normalize by batch size
loss_masked_not_reduced_scalled.sum() / len(loss_masked_not_reduced_scalled)
loss_not_reduced = criterion_red(x, y)
loss_masked_not_reduced = loss_not_reduced * m
loss_masked_not_reduced_scalled = loss_masked_not_reduced / l
loss_masked_not_reduced_scalled.sum() / len(loss_masked_not_reduced_scalled)
loss_masked_not_reduced_scalled<jupyter_output><empty_output><jupyter_text>Loss vectorised<jupyter_code>x.shape == y.shape
x.shape
m.dtype<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1699655765920826, 'fraction_numerical': 0.12908777969018934, 'mean_word_length': 4.260180995475113, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '135487', 'license': 'MIT', 'n_tokens_mistral': 1241, 'n_tokens_neox': 1081, 'n_words': 174, 'path': 'PyTorchNN/BCEWithLogitsLossMasked.ipynb', 'repo_name': 'marcinbogdanski/ai-sketchpad', 'size': '2324'}