<jupyter_start><jupyter_text>1. The binary cross-entropy loss Code example companion notebook for the blog article   [Losses Learned -- Optimizing Negative Log-Likelihood and Cross-Entropy in PyTorch (Part 1)](https://sebastianraschka.com/blog/2022/losses-learned-part1.html)  1.1 Example data for binary cross-entropy<jupyter_code>import torch


y_targets = torch.tensor([1., 1., 0., 0., 0.])

logits = torch.tensor([1.1, 2.2, 0.5, -1.1, -2.2])
probas = torch.sigmoid(logits)
print(probas)<jupyter_output>tensor([0.7503, 0.9002, 0.6225, 0.2497, 0.0998])<jupyter_text>1.2 Implementing the binary cross-entropy loss from scratch<jupyter_code>def binary_logistic_loss_v1(probas, y_targets):
    res = 0.
    for i in range(y_targets.shape[0]):
        if y_targets[i] == 1.:
            res += torch.log(probas[i])
        elif y_targets[i] == 0.:
            res += torch.log(1-probas[i])            
        else:
            raise ValueError(f'Value {y_targets[i]} not allowed')
    res *= -1
    res /= y_targets.shape[0]

    return res


binary_logistic_loss_v1(probas, y_targets)
def binary_logistic_loss_v2(probas, y_targets):
    first = -y_targets.matmul(torch.log(probas))
    second = -(1 - y_targets).matmul(torch.log(1 - probas))
    return (first + second) / y_targets.shape[0]

binary_logistic_loss_v2(probas, y_targets)
%timeit binary_logistic_loss_v1(probas, y_targets)
%timeit binary_logistic_loss_v2(probas, y_targets)<jupyter_output>10.6 µs ± 47.5 ns per loop (mean ± std. dev. of 7 runs, 100,000 loops each)<jupyter_text>1.3 Using the binary cross-entropy loss in PyTorch<jupyter_code>bce = torch.nn.BCELoss()
bce(probas, y_targets)
class MyBCELoss(torch.nn.Module):
    def __init__(self):
        super().__init__()
 
    def forward(self, inputs, targets):        
        return binary_logistic_loss_v2(inputs, targets)
    
    
my_bce = MyBCELoss()
my_bce(probas, y_targets)
bce_logits = torch.nn.BCEWithLogitsLoss()
bce_logits(logits, y_targets)
bce_logits(logits, y_targets)
bce(torch.sigmoid(logits), y_targets)<jupyter_output><empty_output><jupyter_text>**Log-Sum Trick and Logsigmoid**<jupyter_code>def binary_logistic_loss_v2(probas, y_targets):
    first = -y_targets.matmul(torch.log(probas))
    second = -(1 - y_targets).matmul(torch.log(1 - probas))
    return (first + second) / y_targets.shape[0]

binary_logistic_loss_v2(probas, y_targets)
import torch.nn.functional as F


def binary_logistic_loss_v3(logits, y_targets):
    first = -y_targets.matmul(F.logsigmoid(logits))
    second = -(1 - y_targets).matmul(F.logsigmoid(logits) - logits)
    return (first + second) / y_targets.shape[0]

binary_logistic_loss_v3(logits, y_targets)<jupyter_output><empty_output><jupyter_text>1.4 PyTorch’s functional vs object-oriented API<jupyter_code>import torch.nn.functional as F


F.binary_cross_entropy(probas, y_targets)
F.binary_cross_entropy_with_logits(logits, y_targets)<jupyter_output><empty_output><jupyter_text>1.5 A PyTorch loss function cheatsheet (so far) - Note that we use different inputs here, which is why the outputs differ from previous sections.<jupyter_code>logits = torch.tensor([ -1., 0,  1.])
targets = torch.tensor([0.,  0., 1.])

bce_logits = torch.nn.BCEWithLogitsLoss()
bce_logits(logits, targets)
F.binary_cross_entropy_with_logits(logits, targets)
bce = torch.nn.BCELoss()
bce(torch.sigmoid(logits), targets)
F.binary_cross_entropy(torch.sigmoid(logits), targets)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1256150506512301, 'fraction_numerical': 0.029522431259044864, 'mean_word_length': 4.76, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '542271', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1415, 'n_tokens_neox': 1322, 'n_words': 251, 'path': 'losses/pytorch-loss-functions/binary-cross-entropy-in-pytorch.ipynb', 'repo_name': 'rasbt/machine-learning-notes', 'size': '3455'}