<jupyter_start><jupyter_text>PClassifier - A NLP approach for classifying grocery productsThis notebook implements a basic model for classifying grocery products for the GroceryListPredictor project, written by <NAME>, <NAME> und <NAME> for the german national artifical intelligence contest BwKI<jupyter_code># import the libraries needed for the RNN model
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

# some helper classes (see util.py)
from util import randomChoice, lineToTensor, readFile

# for debugging
from IPython.core.debugger import set_trace

# import matplotlib for plotting 
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker

# other libraries required by this notebook
import string
import time
import math
import random
import glob
import os

# the number of hidden neurons of the RNN
n_hidden = 128

# the learning rate
learning_rate = 0.0005

# a list containing all ascii caracters and some german exclusive ones
all_letters = string.ascii_letters + " .,;'" + "äÄüÜöÖß"
n_letters = len(all_letters)

# define the device for the training (Due to a small number of neurons, training on the cpu actually is faster)
device = torch.device("cpu")<jupyter_output><empty_output><jupyter_text>The RNN modelWe initialize a simple self feeding network, which consists of two linear layers. The input and the hidden state are passed through **i2h** and **i2o** and the network returns a new hidden state and a normalized probability distribution consisting of a probability for every category defined.<jupyter_code>class RNN(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(RNN, self).__init__()

        self.hidden_size = hidden_size

        self.i2h = nn.Linear(input_size + hidden_size, hidden_size)
        self.i2o = nn.Linear(input_size + hidden_size, output_size)
        self.softmax = nn.LogSoftmax(dim=1) # softmax_i(x) = log( exp(x_i) / sum(x_z) )  

    def forward(self, input, hidden):
        combined = torch.cat((input, hidden), 1)
        hidden = self.i2h(combined)
        output = self.i2o(combined)
        output = self.softmax(output)
        return output, hidden

    def initHidden(self):
        return torch.zeros(1, self.hidden_size)<jupyter_output><empty_output><jupyter_text>Retrieve the data from the word listsWe read in all the text files from the directory and create the classes and their corresponding elements<jupyter_code>category_lines = {}
categories = []
for filename in glob.glob('../Produktlisten/*.txt'):
    # create the catergories from the filenames
    category = os.path.splitext(os.path.basename(filename))[0] 
    categories.append(category)
    # assign the lines from the file to that category
    lines = readFile(filename) 
    category_lines[category] = lines
n_categories = len(categories)
rnn = RNN(n_letters, n_hidden, n_categories).to(device) # out model
criterion = nn.NLLLoss() # the negative log likelihood loss
optimizer = torch.optim.SGD(rnn.parameters(), lr=0.005)
# returns the prediction of the network by passing every letterTensor through the network
def evaluate(line_tensor):
    hidden = rnn.initHidden() # initialize a hidden tensor for the first feed
    
    # pass every letter tensor through the network
    for i in range(line_tensor.size()[0]):
        output, hidden = rnn(line_tensor[i], hidden)

    return output
rnn.load_state_dict(torch.load("state_dicts/model.pt")) # loads a prtrained model
# returns a random training example
def randomTrainingExample():
    category = randomChoice(categories)
    line = randomChoice(category_lines[category])

    category_tensor = torch.tensor([categories.index(category)], dtype=torch.long)

    line_tensor = lineToTensor(line)
    return category, line, category_tensor, line_tensor<jupyter_output><empty_output><jupyter_text>Train the network<jupyter_code># performs a training step with the given line and category tensor
def train(category_tensor, line_tensor):
    
    hidden = rnn.initHidden() # initialize a hidden tensor for the first feed

    optimizer.zero_grad() # reset the current gradients 
    
    # pass every letter tensor through the network
    for i in range(line_tensor.size()[0]):
        output, hidden = rnn(line_tensor[i], hidden)
    
    # define our loss criterion
    loss = criterion(output, category_tensor)
    loss.backward()

   
    # Add parameters gradients to their values, multiplied by learning rate (Gradient Descent)
    optimizer.step()

    return output, loss.item()
# retieves the category of an output by the network
def categoryFromOutput(output):
    top_n, top_i = output.topk(1)
    category_i = top_i[0].item()
    return categories[category_i], category_i<jupyter_output><empty_output><jupyter_text>Main Loop<jupyter_code>n_iters = 150000
print_every = 5000
plot_every = 1000

current_loss = 0
all_losses = []

def timeSince(since):
    now = time.time()
    s = now - since
    m = math.floor(s / 60)
    s -= m * 60
    return '%dm %ds' % (m, s)

start = time.time()

for iter in range(1, n_iters + 1):
    category, line, category_tensor, line_tensor = randomTrainingExample()
    output, loss = train(category_tensor, line_tensor)
    current_loss += loss
    
    # the logSoftmax function sometimes returns nan, hence the loss is nan. Still haven't found the cause :/
    if np.isnan(loss):
        print("Got NaN at iteration " + str(iter))
        set_trace()
        break

    # Print iter number, loss, name and guess
    if iter % print_every == 0:
        guess, guess_i = categoryFromOutput(output)
        correct = '✓' if guess == category else '✗ (%s)' % category
        print('%d %d%% (%s) %.4f %s / %s %s' % (iter, iter / n_iters * 100, timeSince(start), loss, line, guess, correct))

    # Add current loss avg to list of losses
    if iter % plot_every == 0:
        all_losses.append(current_loss / plot_every)
        current_loss = 0

# plot the graph of the losses returned during training
plt.figure()
plt.plot(all_losses)<jupyter_output>5000 3% (0m 4s) 0.1174 GEFLÜGELFLEISCH / fleisch ✓
10000 6% (0m 8s) 1.5375 KNOBLAUCH / gewuerze ✗ (gemuese)
Got NaN at iteration 13870
> [0;32m<ipython-input-55-1961f7bfe7b2>[0m(26)[0;36m<module>[0;34m()[0m
[0;32m     24 [0;31m        [0mprint[0m[0;34m([0m[0;34m"Got NaN at iteration "[0m [0;34m+[0m [0mstr[0m[0;34m([0m[0miter[0m[0;34m)[0m[0;34m)[0m[0;34m[0m[0;34m[0m[0m
[0m[0;32m     25 [0;31m        [0mset_trace[0m[0;34m([0m[0;34m)[0m[0;34m[0m[0;34m[0m[0m
[0m[0;32m---> 26 [0;31m        [0;32mbreak[0m[0;34m[0m[0;34m[0m[0m
[0m[0;32m     27 [0;31m[0;34m[0m[0m
[0m[0;32m     28 [0;31m    [0;31m# Print iter number, loss, name and guess[0m[0;34m[0m[0;34m[0m[0;34m[0m[0m
[0m
--KeyboardInterrupt--

KeyboardInterrupt: Interrupted by user<jupyter_text>Evaluate the result<jupyter_code>def predict(input_line, n_predictions=3):
    print('\n> %s' % input_line)
    with torch.no_grad():
        output = evaluate(lineToTensor(input_line))

        # Get top N categories
        topv, topi = output.topk(n_predictions, 1, True)
        predictions = []

        for i in range(n_predictions):
            value = topv[0][i].item()
            category_index = topi[0][i].item()
            print('(%.2f) %s' % (value, categories[category_index]))
            predictions.append([value, categories[category_index]])
predict("Ventilator".upper())
torch.save(rnn.state_dict(), "state_dicts/model.pt") # save the trained model<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11399920833883098, 'fraction_numerical': 0.032853938514315874, 'mean_word_length': 4.087248322147651, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '276655', 'license': 'MIT', 'n_tokens_mistral': 2701, 'n_tokens_neox': 2478, 'n_words': 700, 'path': 'old_stuff/pclassifier/PClassifier.ipynb', 'repo_name': 'SimonSkade/ShoppingListPredictor', 'size': '7597'}