<jupyter_start><jupyter_text><jupyter_code>import torch
import numpy as np<jupyter_output><empty_output><jupyter_text>![Cauch-Schwarz Divergence](https://raw.githubusercontent.com/MistreanuIonutCosmin/ML-Path/master/assets/CauchSchwarzDivergence.png)<jupyter_code>device = 'cuda'
def encodeOneHot(tensor):
    a = tensor.cpu().numpy()
    b = np.zeros((a.size, 10))
    b[np.arange(a.size), a] = 1
    return torch.from_numpy(b).float().to(device)

class CauchySchawarzDivergence(torch.nn.Module):
    def __init__(self):
        super(CauchySchawarzDivergence, self).__init__()

    def forward(self, outputs, target):
        y = encodeOneHot(target)
        nom = torch.sum(torch.mm(outputs, y.t()), dim = 1)
        denom = torch.norm(outputs, 2) * torch.norm(y, 2)
        return torch.mean(-1 * torch.log(nom/denom))

b = np.zeros((a.size, 10))
b
 b[[0,1], [9,2]] = 1
 b
## Note: For every other custom function that is not continuous, you must define 
## a backward function as well
optimizer = torch.optim.Adam(network.parameters(), lr=0.0005)
criterion = CauchySchawarzDivergenceCSD()

epoch = 128
steps = len(data_loader) // BATCH_SIZE

network.train(True)

for e in range(epochs):
    epoch_loss = 0
    performed_steps = 0

    for i, (images, labels) in enumerate(train_loader):
        if i == steps:
            break
        images, labels = images.to(device), labels.to(device)
        optimizer.zero_grad()
        outputs = network(images)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
        epoch_loss += loss
        performed_steps += 1
        epoch_loss / steps
avg_loss = 0
avg_acc = 0

network.train(False)

with torch.no_grad():
    steps = 0
    
    for images, labels in test_loader:
        images, labels = images.to(device), labels.to(deivce)
        outputs = network(images)
        avg_loss += criterion(outputs, labels)
        _, preds = torch.max(outputs, 1)
        avg_acc += preds.g(labels).sum().item()
        steps += 1<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13130818226359628, 'fraction_numerical': 0.014698677119059285, 'mean_word_length': 3.227743271221532, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '563959', 'license': 'MIT', 'n_tokens_mistral': 732, 'n_tokens_neox': 703, 'n_words': 148, 'path': 'implementations/CauchySchawarzDivergence.ipynb', 'repo_name': 'MistreanuIonutCosmin/ML-Path', 'size': '2041'}