<jupyter_start><jupyter_text>Implementation of pytorch network   Imports<jupyter_code>import os
import torch
import mlflow
import numpy as np
import pandas as pd
import plotly.express as px
from itertools import product
from torch import nn, cuda, optim, no_grad
import torch.nn.functional as F
from torchvision import transforms
from torchvision.datasets import ImageFolder<jupyter_output><empty_output><jupyter_text>Data loaders<jupyter_code>def load_plant_data(resolution=32):
    """This will load the data and transform it."""
    base_dir = os.path.join(os.pardir, 'data', f'Plant_leave_diseases_{resolution}')
    train_dir = os.path.join(base_dir, 'train')
    validation_dir = os.path.join(base_dir, 'validation')

    data_transform = transforms.Compose([
                    transforms.Grayscale(num_output_channels=1),
                    transforms.ToTensor(),
                    transforms.Normalize((0.5), (0.5))
                ])

    train_dataset = ImageFolder(root=train_dir, transform=data_transform)
    validation_dataset = ImageFolder(root=validation_dir, transform=data_transform)

    if os.path.isdir(os.path.join(base_dir, 'test')):
        test_dir = os.path.join(base_dir, 'test')
        test_dataset = ImageFolder(root=test_dir, transform=data_transform)
        return train_dataset, test_dataset, validation_dataset
    return train_dataset, validation_dataset<jupyter_output><empty_output><jupyter_text>Neural Net<jupyter_code>class PlantDiseaseNet(nn.Module):
    def __init__(self, input_size=1024, l1=1024, l2=512, output_size=39, dropout_p=0.5):
        super(PlantDiseaseNet, self).__init__()
        self.fc1 = nn.Linear(input_size, l1)
        self.fc2 = nn.Linear(l1, l2)
        self.fc3 = nn.Linear(l2, output_size)
        self.dropout = nn.Dropout(dropout_p)

    def forward(self, x):
        x = x.view(x.shape[0], -1)
        x = F.relu(self.fc1(x))
        x = self.dropout(x)
        x = F.relu(self.fc2(x))
        x = self.dropout(x)
        x = F.log_softmax(self.fc3(x), dim=1)
        return x<jupyter_output><empty_output><jupyter_text>Training function<jupyter_code>def train(train_loader, validation_loader, config, n_epochs=10, stopping_treshold=None):

    
    # Initializing the model
    model = PlantDiseaseNet(input_size=config['resolution']**2, dropout_p=config['dropout'])
    print("Starting training on network: \n", model)
    # Checking machine resources available

    if cuda.is_available():
        print('CUDA is available!  Training on GPU ...')
        model.cuda()


    # Loss and optimizer setup 
    criterion = nn.NLLLoss()
    optimizer = optim.Adam(model.parameters(), lr=config['learning_rate'], weight_decay=config['decay'])

    # Setting minimum validation loss to inf
    validation_loss_minimum = np.Inf 
    train_loss_history = []
    validation_loss_history = []

    for epoch in range(1, n_epochs +1):

        training_loss = 0.0
        validation_loss = 0.0

        # Training loop
        training_accuracies = []
        for X, y in train_loader:
            
            # Moving data to gpu if using 
            if cuda.is_available():
                X, y = X.cuda(), y.cuda()
            
            # clear the gradients of all optimized variables
            optimizer.zero_grad()
            # forward pass: compute predicted outputs by passing inputs to the model
            output = model(X)
            # calculate the batch loss
            loss = criterion(output, y)
            # backward pass: compute gradient of the loss with respect to model parameters
            loss.backward()
            # perform a single optimization step (parameter update)
            optimizer.step()
            # update training loss
            training_loss += loss.item()*X.size(0)

            # calculating accuracy
            ps = torch.exp(output)
            top_p, top_class = ps.topk(1, dim=1)
            equals = top_class == y.view(*top_class.shape)
            training_accuracies.append(torch.mean(equals.type(torch.FloatTensor)).item())

        # Validation Loop
        with no_grad():
            accuracies = []
            for X, y in validation_loader:

                # Moving data to gpu if using 
                if cuda.is_available():
                    X, y = X.cuda(), y.cuda()
                # forward pass: compute predicted outputs by passing inputs to the model
                output = model(X)
                # calculate the batch loss
                loss = criterion(output, y)
                # update validation loss
                validation_loss += loss.item()*X.size(0)

                # calculating accuracy
                ps = torch.exp(output)
                top_p, top_class = ps.topk(1, dim=1)
                equals = top_class == y.view(*top_class.shape)
                accuracies.append(torch.mean(equals.type(torch.FloatTensor)).item())
                
        # Mean loss 
        mean_training_loss = training_loss/len(train_loader.sampler)
        mean_validation_loss = validation_loss/len(validation_loader.sampler)
        mean_train_accuracy = sum(training_accuracies)/len(training_accuracies)
        mean_accuracy = sum(accuracies)/len(accuracies)
        train_loss_history.append(mean_training_loss)
        validation_loss_history.append(mean_validation_loss)

        # Printing epoch stats
        print(f'Epoch: {epoch}/{n_epochs}, ' +\
              f'Training Loss: {mean_training_loss:.3f}, '+\
              f'Train accuracy {mean_train_accuracy:.3f} ' +\
              f'Validation Loss: {mean_validation_loss:.3f}, '+\
              f'Validation accuracy {mean_accuracy:.3f}')

        # logging with mlflow 
        if mlflow.active_run():
            mlflow.log_metric('loss', mean_training_loss, step=epoch)
            mlflow.log_metric('accuracy', mean_train_accuracy, step=epoch)
            mlflow.log_metric('validation_accuracy', mean_accuracy, step=epoch)
            mlflow.log_metric('validation_loss', mean_validation_loss, step=epoch)

        # Testing for early stopping
        if stopping_treshold:
            if mean_validation_loss < validation_loss_minimum:
                validation_loss_minimum = mean_validation_loss
            elif len([v for v in validation_loss_history[-stopping_treshold:] if v > validation_loss_minimum]) >= stopping_treshold:
                print(f"Stopping early at epoch: {epoch}/{n_epochs}")
                break
        

    return train_loss_history, validation_loss_history<jupyter_output><empty_output><jupyter_text>Setting up experiments<jupyter_code># Parimiters to test 
resolutions = [32]
leanring_rate = [0.001, 0.002]
decay = [1e-4, 1e-3]
dropout = [0.2, 0.9, 0.5]
configs = [{
    'resolution': cfg[0],
    'learning_rate': cfg[1],
    'decay': cfg[2],
    'dropout': cfg[3],
    'max_epochs': 200
    
} for cfg in product(resolutions, leanring_rate, decay, dropout)]
for config in configs:
        
    # Set up data loaders
    train_ds, validate_ds = load_plant_data(resolution=config['resolution'])
    train_loader = torch.utils.data.DataLoader(train_ds, batch_size=64, shuffle=True)
    validation_loader = torch.utils.data.DataLoader(validate_ds, batch_size=64, shuffle=True)

    mlflow.set_experiment("Plant Leaf Disease")

    with mlflow.start_run():
        mlflow.log_param('framework', 'pytorch')
        mlflow.log_param('data_split', '90/10')
        mlflow.log_param('type', 'FFNN')
        mlflow.log_params(config)
        tlh, vlh = train(train_loader, validation_loader, config, n_epochs=config['max_epochs'], stopping_treshold=50)
        config['train_loss_history'] = tlh
        config['validation_loss_history'] = vlh<jupyter_output>Starting training on network: 
 PlantDiseaseNet(
  (fc1): Linear(in_features=1024, out_features=1024, bias=True)
  (fc2): Linear(in_features=1024, out_features=512, bias=True)
  (fc3): Linear(in_features=512, out_features=39, bias=True)
  (dropout): Dropout(p=0.2, inplace=False)
)
CUDA is available!  Training on GPU ...
Epoch: 1/200, Training Loss: 2.170, Train accuracy 0.397 Validation Loss: 1.923, Validation accuracy 0.453
Epoch: 2/200, Training Loss: 1.698, Train accuracy 0.507 Validation Loss: 1.676, Validation accuracy 0.512
Epoch: 3/200, Training Loss: 1.481, Train accuracy 0.563 Validation Loss: 1.579, Validation accuracy 0.537
Epoch: 4/200, Training Loss: 1.351, Train accuracy 0.596 Validation Loss: 1.485, Validation accuracy 0.558
Epoch: 5/200, Training Loss: 1.245, Train accuracy 0.623 Validation Loss: 1.481, Validation accuracy 0.574
Epoch: 6/200, Training Loss: 1.165, Train accuracy 0.643 Validation Loss: 1.430, Validation accuracy 0.581
Epoch: 7/200, Training Loss: 1.102, [...]
{'fraction_non_alphanumeric': 0.10528730358986123, 'fraction_numerical': 0.028214244752838628, 'mean_word_length': 2.891120035698349, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '72129', 'license': 'MIT', 'n_tokens_mistral': 2880, 'n_tokens_neox': 2652, 'n_words': 607, 'path': 'notebooks/FFNN_pytorch.ipynb', 'repo_name': 'NGrech/plant-leaf-diseases-identification', 'size': '8719'}