<jupyter_start><jupyter_text>Activation Functions  Sigmoid<jupyter_code>import torch
import matplotlib.pyplot as plt

x = torch.range(-5., 5., 0.1)
y = torch.sigmoid(x)
plt.plot(x.numpy(), y.numpy())
plt.show()<jupyter_output>C:\ProgramData\Anaconda3\envs\only_flair\lib\site-packages\ipykernel_launcher.py:4: UserWarning: torch.range is deprecated in favor of torch.arange and will be removed in 0.5. Note that arange generates values in [start; end), not [start; end].
  after removing the cwd from sys.path.<jupyter_text>Tanh<jupyter_code>import torch
import matplotlib.pyplot as plt

x = torch.range(-5.,5., 0.1)
y = torch.tanh(x)
plt.plot(x.numpy(), y.numpy())
plt.show()<jupyter_output>C:\ProgramData\Anaconda3\envs\only_flair\lib\site-packages\ipykernel_launcher.py:4: UserWarning: torch.range is deprecated in favor of torch.arange and will be removed in 0.5. Note that arange generates values in [start; end), not [start; end].
  after removing the cwd from sys.path.<jupyter_text>ReLU<jupyter_code>import torch
import matplotlib.pyplot as plt

relu = torch.nn.ReLU()
x = torch.range(-5.,5., 0.1)
y = relu(x)
plt.plot(x.numpy(), y.numpy())
plt.show()<jupyter_output>C:\ProgramData\Anaconda3\envs\only_flair\lib\site-packages\ipykernel_launcher.py:5: UserWarning: torch.range is deprecated in favor of torch.arange and will be removed in 0.5. Note that arange generates values in [start; end), not [start; end].
  """<jupyter_text>PReLU<jupyter_code>import torch
import matplotlib.pyplot as plt

prelu = torch.nn.PReLU(num_parameters=1)
x = torch.range(-5., 5., 0.1)
y = prelu(x)
plt.plot(x.numpy(), y.detach().numpy())
plt.show()<jupyter_output>C:\ProgramData\Anaconda3\envs\only_flair\lib\site-packages\ipykernel_launcher.py:5: UserWarning: torch.range is deprecated in favor of torch.arange and will be removed in 0.5. Note that arange generates values in [start; end), not [start; end].
  """<jupyter_text>Softmax<jupyter_code>import torch.nn as nn
import torch

softmax = nn.Softmax(dim=1)
x_input = torch.randn(1, 3)
y_output = softmax(x_input)
print(x_input)
print(y_output)
print(torch.sum(y_output, dim=1))<jupyter_output>tensor([[ 1.3450, -1.7756, -0.3565]])
tensor([[0.8153, 0.0360, 0.1487]])
tensor([1.])<jupyter_text>Loss Functions  Mean Squared Error Loss<jupyter_code>import torch
import torch.nn as nn

mse_loss = nn.MSELoss()
outputs = torch.randn(3, 5, requires_grad=True)
targets = torch.randn(3, 5)
loss = mse_loss(outputs, targets)
print(loss)<jupyter_output>tensor(2.2737, grad_fn=<MseLossBackward>)<jupyter_text>Categorical Cross-Entropy Loss<jupyter_code>import torch
import torch.nn as nn

ce_loss = nn.CrossEntropyLoss()
outputs = torch.randn(3, 5, requires_grad=True)
targets = torch.tensor([1, 0, 3], dtype=torch.int64)
loss = ce_loss(outputs, targets)
print(loss)<jupyter_output>tensor(2.4838, grad_fn=<NllLossBackward>)<jupyter_text>Binary Cross-Entropy Loss<jupyter_code>bce_loss = nn.BCELoss()
sigmoid = nn.Sigmoid()
probabilities = sigmoid(torch.randn(4, 1, requires_grad=True))
targets = torch.tensor([1, 0, 1, 0], dtype=torch.float32).view(4, 1)
loss = bce_loss(probabilities, targets)
print(probabilities)
print(loss)<jupyter_output>tensor([[0.6496],
        [0.5403],
        [0.4998],
        [0.1918]], grad_fn=<SigmoidBackward>)
tensor(0.5288, grad_fn=<BinaryCrossEntropyBackward>)
{'fraction_non_alphanumeric': 0.14328627215379994, 'fraction_numerical': 0.037248422949834784, 'mean_word_length': 5.894409937888199, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '52205', 'license': 'Apache-2.0', 'n_tokens_mistral': 1383, 'n_tokens_neox': 1305, 'n_words': 272, 'path': 'chapters/chapter_3/Chapter 3.ipynb', 'repo_name': 'e93fem/PyTorchNLPBook', 'size': '3329'}