<jupyter_start><jupyter_text>Intro Deep Learning<jupyter_code>import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
torch.manual_seed(1)
lin = nn.Linear(5, 3)# maps from R^5 to R^3, parameters A, b
# data is 2x5. A maps from 5 to 3.. can we map "data" under A?
data = torch.randn(2, 5)
print(lin(data))
data = torch.randn(2, 2)
print(data)
print(F.relu(data))
data = torch.randn(5)
print(data)
print(F.softmax(data, dim=0))
print(F.softmax(data, dim=0).sum())
print(F.log_softmax(data, dim=0))
print(F.log_softmax(data, dim=0).sum())<jupyter_output>tensor([ 1.3800, -1.3505,  0.3455,  0.5046,  1.8213])
tensor([0.2948, 0.0192, 0.1048, 0.1228, 0.4584])
tensor(1.)
tensor([-1.2214, -3.9519, -2.2560, -2.0969, -0.7801])
tensor(-10.3063)<jupyter_text>Declaring Model<jupyter_code>data = [("me gusta comer en la cafeteria".split(), "SPANISH"),
        ("Give it to me".split(), "ENGLISH"),
        ("No creo que sea una buena idea".split(), "SPANISH"),
        ("No it is not a good idea to get lost at sea".split(), "ENGLISH")]

test_data = [("Yo creo que si".split(), "SPANISH"),
             ("it is lost on me".split(), "ENGLISH")]

word_to_idx = {}
for sent, _ in data + test_data:
  for word in sent:
    if word not in word_to_idx:
      word_to_idx[word] = len(word_to_idx)
print(word_to_idx)

VOCAB_SIZE = len(word_to_idx)
NUM_LABELS = 2

class BoWClassifier(nn.Module):
  def __init__(self, num_labels, vocab_size):
    super(BoWClassifier, self).__init__()

    self.linear = nn.Linear(vocab_size, num_labels)

  def forward(self, bow_vec):
    return F.log_softmax(self.linear(bow_vec), dim=1)

def make_bow_vector(sentence, word_to_idx):
  vec = torch.zeros(len(word_to_idx))
  for word in sentence:
    vec[word_to_idx[word]] = vec[word_to_idx[word]] + 1
  return vec.view(1, -1)

def make_target(label, label_to_idx):
  return torch.LongTensor([label_to_idx[label]])

model = BoWClassifier(NUM_LABELS, VOCAB_SIZE)
for param in model.parameters():
  print(param)

# Take if the model works
with torch.no_grad():
  sample = data[0]
  bow_vector = make_bow_vector(sample[0], word_to_idx)
  log_probs = model(bow_vector)
  print(log_probs)
label_to_idx = {"SPANISH": 0, "ENGLISH": 1}
with torch.no_grad():
  for instance, label in test_data:
    bow_vec = make_bow_vector(instance, word_to_idx)
    log_probs = model(bow_vec)
    print(log_probs)

# Print the matrix column corresponding to "creo"
print(next(model.parameters())[:, word_to_idx["creo"]])

loss_fn = nn.NLLLoss()
optimizer = optim.SGD(model.parameters(), lr=0.1)
for epoch in range(100):
  for instance, label in data:
    # Step 1. Remember that PyTorch accumulates gradients.
    # We need to clear them out before each instance
    model.zero_grad()

    # Step 2. Make our BOW vector and also we must wrap the target in a
    # Tensor as an integer. For example, if the target is SPANISH, then
    # we wrap the integer 0. The loss function then knows that the 0th
    # element of the log probabilities is the log probability
    # corresponding to SPANISH
    bow_vec = make_bow_vector(instance, word_to_idx)
    target = make_target(label, label_to_idx)

    # Step 3. Run our forward pass.
    log_probs = model(bow_vec)
    # Step 4. Compute the loss, gradients, and update the parameters by
    # calling optimizer.step()
    loss = loss_fn(log_probs, target)
    loss.backward()
    optimizer.step()

with torch.no_grad():
  for instance, label in test_data:
    bow_vec = make_bow_vector(instance, word_to_idx)
    log_probs = model(bow_vec)
    print(log_probs)

# Index corresponding to Spanish goes up, English goes down!
print(next(model.parameters())[:, word_to_idx["creo"]])<jupyter_output>tensor([[-0.9297, -0.5020]])
tensor([[-0.6388, -0.7506]])
tensor([-0.1488, -0.1313], grad_fn=<SelectBackward0>)
tensor([[-0.2093, -1.6669]])
tensor([[-2.5330, -0.0828]])
tensor([ 0.2803, -0.5605], grad_fn=<SelectBackward0>)
{'fraction_non_alphanumeric': 0.13999492772001015, 'fraction_numerical': 0.0461577479076845, 'mean_word_length': 4.315363881401617, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '358400', 'license': 'MIT', 'n_tokens_mistral': 1584, 'n_tokens_neox': 1497, 'n_words': 388, 'path': 'DL_PyTorch.ipynb', 'repo_name': 'TokisakiKurumi2001/pytorch_nlp_learn', 'size': '3943'}