<jupyter_start><jupyter_text>Class Proportions (for Weighted Sampling)<jupyter_code># Get the class proportions
class_counts = np.bincount(y_validation.astype(int))
total = len(y_validation)
proportion_0 = class_counts[0] / total
proportion_1 = class_counts[1] / total
proportion_2 = class_counts[2] / total

print('Class percentages:\nNo fog: {:.2f}%\nFog: {:.2f}%\nDense fog: {:.2f}%'.format(proportion_0 * 100,
                                                                              proportion_1 * 100, proportion_2 * 100))
print(class_counts)<jupyter_output><empty_output><jupyter_text>Class Weighting<jupyter_code># List containing class probabilities
probabilities = [proportion_0, proportion_1, proportion_2]
reciprocal_weights = []

# Put weight at every index
for i in range(len(X_train)):
    reciprocal_weights.append(probabilities[train_targets[i]])

# Inverse of probabilities as weights
weights = (1 / torch.Tensor(reciprocal_weights))
sampler = torch.utils.data.sampler.WeightedRandomSampler(weights.double(), len(X_train))

# Inverse weights for all the datapoints
inverse_weights_class = 1 / torch.Tensor(probabilities)

# Inverse weights per class
inverse_weights = 1 / torch.Tensor(probabilities)<jupyter_output><empty_output><jupyter_text>Transformations<jupyter_code>data_transforms = {
    'train': transforms.Compose([
        transforms.ToPILImage(),
        transforms.RandomCrop(80),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
    'validation': transforms.Compose([
        transforms.ToPILImage(),
        transforms.CenterCrop(80),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
}<jupyter_output><empty_output><jupyter_text>Define Dataset Class<jupyter_code>class KNMIDataset(Dataset):
    def __init__(self, images, targets, filepaths, transforms=None):
    
        self.transforms = transforms
        self.images = images
        self.targets = targets
        self.filepaths = filepaths
        
    def __getitem__(self, index):
        image = self.images[index]
        
        if self.transforms != None:
            image = self.transforms(image)
        
        target = self.targets[index]
        filepath = self.filepaths[index]
        
        return (image, target, index, filepath)

    def __len__(self):
        return len(self.targets)<jupyter_output><empty_output><jupyter_text>Create Datasets and Dataloaders<jupyter_code>BATCH_SIZE = 164

# Datasets
train_dataset = KNMIDataset(X_train, train_targets, paths_train, transforms=data_transforms['train'])
validation_dataset = KNMIDataset(X_validation, validation_targets, paths_validation, transforms=data_transforms['validation'])

# Data loaders
train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True)
validation_loader = DataLoader(dataset=validation_dataset, batch_size=BATCH_SIZE)

loaders = {'train': train_loader, 'validation': validation_loader}<jupyter_output><empty_output><jupyter_text>Check Batch Iteration Size of Trainloader<jupyter_code># Iteration for one train/testloader batch
img, labels, idx, paths = next(iter(validation_loader))
inputs, labels = Variable(img), Variable(labels)
print('Loader image tensor shape: {}\nLoader targets tensor shape: {}'.format(inputs.size(), labels.size()))<jupyter_output><empty_output><jupyter_text>Plotting Functions  Confusion Matrix<jupyter_code># Confusion matrix helper
def show_cm(targets, predictions):
    cm = confusion_matrix(y_target=targets, 
                      y_predicted=predictions, 
                      binary=False)

    fig, ax = plot_confusion_matrix(conf_mat=cm)
    plt.show()<jupyter_output><empty_output><jupyter_text>Loss Curve Plotting<jupyter_code>def plot_loss_curves(training_loss, validation_loss):
    """
    Plots loss curves after model training.
    
    :param training_loss: List with training loss for every epoch.
    :param validation_loss: List with validation loss for every epoch.
    """
    train_plot, = plt.plot(training_loss, label='Training')
    val_plot, = plt.plot(validation_loss, label='Validation')
    plt.title('Loss curves (training/validation)')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.legend(handles=[train_plot, val_plot])
    plt.show()<jupyter_output><empty_output><jupyter_text>Certain/Uncertain Images Plotting<jupyter_code>def plot_images(loss, image_index, filepaths, targets, predictions, phase, amount=5):
    """
    Use to plot images that the model is most certain about and which it was most uncertain about.
    
    :param loss: Tensor that has size of batch containing loss
    :param filepaths: List with filepaths that point to where batch images are located
    :param amount: Amount of images to show. Default: 1
    """
    
    def loop_plot(indices, targets, predictions, filepaths, losses, phase):    
        fig=plt.figure(figsize=(20, 5))
        columns = 5
        rows = 1
        
        # Determine phase and get image np array
        if phase == 'train':
            image_array = X_train
        elif phase == 'validation':
            image_array = X_validation
        else:
            image_array = test_images
        print(filepaths)
        # Loop over the data and plot 'amount' of images
        for i, (index, target, prediction, loss) in enumerate(zip(indices, targets, predictions, losses)):
            img = image_array[index]
            fig.add_subplot(rows, columns, i + 1)
            plt.title('target: {}, prediction: {} loss: {:.2f}'.format(target, prediction, loss))
            plt.imshow(img)
      
        plt.show()
    
    def get_k_and_plot(loss, amount, targets, image_index, filepaths, predictions, phase, largest=True):
        # Get all relevant data
        values, indices = torch.topk(loss, amount, largest=largest)
        targets = [targets[i].data[0] for i in list(indices.data.numpy().reshape((1, -1))[0])]
        images_idx = [image_index[i] for i in list(indices.data.numpy().reshape((1, -1))[0])]
        filepaths = [filepaths[i] for i in list(indices.data.numpy().reshape((1, -1))[0])]
        predictions = [predictions[i] for i in list(indices.data.numpy().reshape((1, -1))[0])]
        loss = [loss.data[i] for i in list(indices.data.numpy().reshape((1, -1))[0])]

        # Show images (uncertain/certain)
        loop_plot(images_idx, targets, predictions, filepaths, loss, phase)

    print('Top {} most uncertain images'.format(amount))
    get_k_and_plot(loss, amount, targets, image_index, filepaths, predictions, phase, largest=True)

    print('Top {} most certain images'.format(amount))
    get_k_and_plot(loss, amount, targets, image_index, filepaths, predictions, phase, largest=False)<jupyter_output><empty_output><jupyter_text>Model Training  Main Training Function<jupyter_code>def train_model(model, criterion, optimizer, num_epochs):
    """
    Does the actual training of the models.
    
    :param model: Model object specified in 'run_model'.
    :param criterion: Optimization criterion/loss.
    :param optimizer: Type of optimizer.
    :param num_epochs: Number of epochs to train.
    """
    start = time.time()

    # For storing loss for curve, best model and best accuracy
    train_loss, validation_loss = [],[]
    best_model = model
    best_accuracy = 0.0
    best_f1macro = 0.0

    for epoch in range(num_epochs):

        # Running loss and correct predictions
        running_loss_train = 0.0
        running_correct_train = 0.0
        running_loss_val = 0.0
        running_correct_val = 0.0
        epoch_validation_targets = []
        epoch_validation_predictions = []

        for phase in ['train', 'validation']:

            # Change model mode according to phase 
            if phase == 'train':
                model.train()
            else:
                model.eval()

            # Iterate over batches in loader
            for i, (image_tensor, label_tensor, image_index, filepaths) in enumerate(loaders[phase]):

                features = Variable(image_tensor)
                targets = Variable(label_tensor.view(-1))

                # Forward + Backward + Optimize
                if phase == 'train':
                    optimizer.zero_grad()

                outputs = model(features)

                # Get prediction index and no. correct predictions
                _, predictions = torch.max(outputs.data, 1) 
                
                # Kijk hier uit dat die testloader precies aantal batches in de validation data haalt 
                correct = torch.sum(predictions == targets.data) 

                # Loss and optimization
                loss = criterion(outputs, targets)
                
                # Average the loss
                total_loss = torch.mean(loss)
                
                # Only do backpropagation if in the training phase
                if phase == 'train':
                    total_loss.backward()
                    optimizer.step()

                # Running loss and number of correct predictions
                if phase == 'train':
                    running_loss_train += total_loss.data[0]
                    running_correct_train += correct
                else:
                    running_loss_val += total_loss.data[0]
                    running_correct_val += correct
                    epoch_validation_targets.extend(list(targets.data))
                    epoch_validation_predictions.extend(list(predictions))
                    
                    # Plot images in validation phase
                    plot_images(loss, image_index, filepaths, targets, predictions, phase)

                # If model is in training phase, show loss every N iterations
                if (i+1) % 5 == 0:
                    if phase == 'train':
                        print ('Epoch {}/{}, Iteration {}/{} Train Running Loss: {:.4f}'.format(epoch+1, num_epochs, i+1, 
                                                                                    len(X_train)//BATCH_SIZE, 
                                                                                    running_loss_train / i))
#                         plot_images(loss, image_index, filepaths, targets, predictions, phase)
                    

        # Epoch losses and epoch train accuracies
        epoch_train_loss = running_loss_train / (len(X_train)//BATCH_SIZE)
        epoch_train_accuracy = (running_correct_train / (len(X_train)//BATCH_SIZE)) / BATCH_SIZE * 100
        epoch_val_loss = running_loss_val / len(X_validation//BATCH_SIZE)
        epoch_val_accuracy= running_correct_val / len(X_validation) * 100
        
        # F1-score
        f1_macro = f1_score(epoch_validation_targets, epoch_validation_predictions, average='macro')
        f1_micro = f1_score(epoch_validation_targets, epoch_validation_predictions, average='micro')
        precision = precision_score(epoch_validation_targets, epoch_validation_predictions, average='macro')
        recall = recall_score(epoch_validation_targets, epoch_validation_predictions, average='macro')

        # Print the average epoch loss and the average prediction accuracy
        print('\nEpoch {}/{}, Train Loss: {:.4f}, Train Accuracy: {:.4f}%\n'
              'Validation Loss: {:.4f}, Validation Accuracy: {:.4f}%, f1-score {:.4f}\n'.format(epoch + 1, 
                                                                    num_epochs, epoch_train_loss, epoch_train_accuracy,
                                                                     epoch_val_loss, epoch_val_accuracy, f1))
        
        # Safe best model and best accuracy 
        if phase == 'validation':
            if epoch_val_accuracy > best_accuracy:
                best_accuracy = epoch_val_accuracy
                best_model = copy.deepcopy(model)

        # Show the confusion matrix for validation targets/predictions
        show_cm(epoch_validation_targets, epoch_validation_predictions)
        
        # Append losses
        train_loss.append(epoch_train_loss)
        validation_loss.append(epoch_val_loss)
        
    # Elapsed time and best accuracy
    elapsed_time = time.time() - start
    print('Training was completed in {:.0f}m {:.0f}s\n'.format(elapsed_time//60, elapsed_time%60))
    print('Best validation accuracy: {:4f}%'.format(best_accuracy))
    
    # Plot loss curves
    plot_loss_curves(train_loss, validation_loss)
    
    # Return the best model
    return best_model<jupyter_output><empty_output><jupyter_text>Run Model Function<jupyter_code>def run_model(model, epochs, learning_rate, train_from_layer=False, last_layer_trained=False, not_self_defined=True):
    '''
    Configures model object and then calls 'train_model' for model training. 
    
    :param train_from_layer: Specify number of layers before fully-connected to also be finetuned. Default is False,
    which will just train the fc layers. Give a number to specify number of layers before that. 
    :param model: This is a (pre-trained) model that will be further finetuned.
    :param epochs: Number of epochs to train.
    :param learning_rate: Learning rate for the parameters.
    :param not_self_defined: True if model architecture is used from torchvision. False if model is self-defined. 
    '''
    
    if not_self_defined:
      
        # Set all parameter training to false
        for parameter in model.parameters():
            parameter.requires_grad = False

        # Select number of pre-trained layers to finetune
        if train_from_layer != False:
            ct = 0
            print(ct)

            for name, child in model.named_children():
                ct += 1

                if ct > train_from_layer:
                    for name2, params in child.named_parameters():
                        params.requires_grad = True

          # Get parameters that need finetuning
            optim_params = filter(lambda p: p.requires_grad, model.parameters())

    # Adjust final layer to number of classes if last layer has not been trained yet
    if last_layer_trained == False:
        num_features = model.fc.in_features
        model.fc = nn.Linear(num_features, TARGET_SIZE)
        optim_params = model.fc.parameters()
    
    # Train all parameters if model is not predefined from torchvision
    else: 
        optim_params = filter(lambda p: p.requires_grad, model.parameters())
    
    if torch.cuda.is_available():
        inverse_weights.cuda()
        model = model.cuda()
    
    # Optimizers and loss criterions
    criterion = nn.CrossEntropyLoss(reduce=False, weight=inverse_weights)
    optimizer = optim.Adam(optim_params, lr=learning_rate)        
    
    # Train and save
    trained_model = train_model(model, criterion, optimizer, epochs)
    
    return trained_model<jupyter_output><empty_output><jupyter_text>Resnet 18<jupyter_code>lr_classifier = 2e-4
lr_tuner = 1e-4

# Train last FC layer of resnet 18
resnet_18 = models.resnet18(pretrained=True)
resnet_18.avgpool = nn.AdaptiveAvgPool2d(1)
resnet_18_trained = run_model(resnet_18, 10, lr_classifier, False)

# Tune also the last convolutional layer
# resnet_18_tuned = run_model(resnet_18_trained, 15, lr_tuner, train_from_layer=7)<jupyter_output><empty_output><jupyter_text>Simple Network<jupyter_code>class simple_CNN(nn.Module):
    def __init__(self):
        super(simple_CNN, self).__init__()
        self.conv1 = nn.Sequential(         # input shape (1, 28, 28)
            nn.Conv2d(
                in_channels=3,              # input height
                out_channels=16,            # n_filters
                kernel_size=3,              # filter size
                stride=1,                   # filter movement/step
                padding=2,                  # if want same width and length of this image after con2d, padding=(kernel_size-1)/2 if stride=1
            ),                              # output shape (16, 28, 28)
            nn.ReLU(),                      # activation
            nn.MaxPool2d(kernel_size=2),    # choose max value in 2x2 area, output shape (16, 14, 14)
        )
        self.conv2 = nn.Sequential(         # input shape (16, 14, 14)
            nn.Conv2d(16, 32, 3, 1, 2),     # output shape (32, 14, 14)
            nn.ReLU(),                      # activation
            nn.MaxPool2d(2),                # output shape (32, 7, 7)
        )
        self.conv3 = nn.Sequential(         # input shape (16, 14, 14)
            nn.Conv2d(32, 32, 3, 1, 2),     # output shape (32, 14, 14)
            nn.ReLU(),                      # activation
            nn.MaxPool2d(2),                # output shape (32, 7, 7)
        )
        self.dropout = nn.Dropout(p=0.5)
        self.out = nn.Linear(3872, 3)   # fully connected layer, output 3 classes

    def forward(self, x):
        x = self.conv1(x)
        x = self.conv2(x)
        x = self.conv3(x)
        x = x.view(x.size(0), -1)           # flatten the output of conv2 to (batch_size, 32 * 7 * 7)
        x = self.dropout(x) 
        output = self.out(x)
        return output   # return x for visualization
simple_model = simple_CNN()
run_model(simple_model, 10, 0.001, False, False)<jupyter_output><empty_output><jupyter_text>Pre-trained Model Loader<jupyter_code>def load_model(filepath):
    '''
    Loads a trained model.
    
    :param filepath: Path to the trained model.
    '''
    loaded_model = torch.load(filepath, map_location=lambda storage, loc: storage)
    return loaded_model
TRAINED_MODELS_DIR = '/Volumes/TIMPP/TrainedModels'

loaded_checkpoint = load_model(TRAINED_MODELS_DIR + '/highway-images/resnet-18/Resnet18-FirstBest/classifier/resnet18_classifier.pth.tar')
current_model_tuned = load_model(TRAINED_MODELS_DIR + '/highway-images/resnet-18/Resnet18-FirstBest/tuner/resnet18_tuner.pth.tar')
# current_model_trained = load_model(TRAINED_MODELS_DIR + '/highway-images/resnet18/classifier')
loaded_checkpoint = load_model(TRAINED_MODELS_DIR + '/knmi-images/resnet18/tuner/KNMItuner.pth.tar')
current_model_train = loaded_checkpoint['best_model']
loaded_checkpoint['best_epoch']
def concat_loss_lists(full_train):
    '''
    :param full_train: List of model trained models in different phases.
    '''
    train_loss = []
    validation_loss = []
    
    for train_phase in full_train:
        best_epoch = train_phase['best_epoch']
        phase_train_loss = train_phase['train_loss'][:best_epoch] 
        phase_val_loss = train_phase['validation_loss'][:best_epoch]
        
        for i, j in zip(phase_train_loss, phase_val_loss):
            train_loss.append(i)
            validation_loss.append(j)
    
    return train_loss, validation_loss<jupyter_output><empty_output><jupyter_text>Test Model Function  Test Dataset and Dataloader<jupyter_code>test_dataset = KNMIDataset(test_features, test_targets, test_filepaths, transforms=data_transforms['validation'])
test_dataloader = DataLoader(dataset=test_dataset, batch_size=len(test_features))
loaded_checkpoint = load_model('../../../../Downloads/checkpoint21.pth.tar')
current_model_train = loaded_checkpoint['state_dict']
best_epoch = loaded_checkpoint['best_epoch']
plot_loss_curves(loaded_checkpoint['train_loss'], loaded_checkpoint['validation_loss'])<jupyter_output><empty_output><jupyter_text>Model Test Function<jupyter_code>def test_model(model, dataloader):
    """
    Tests a specified model on all the manually labeled highway camera
    images. 
    
    :param model: Trained model to evaluate
    :param test_features: All test features as tensor
    :param test_targets: All test labels as tensor
    """
    
    test_images, test_targets, idx, test_filepaths = next(iter(dataloader))
    
    # Loss criterion
    criterion = nn.CrossEntropyLoss(reduce=False)
    
    # Wrap tensors
    features = Variable(test_images)
    targets = Variable(test_targets)
    total = len(targets)
    
    model.eval()
    
    # Feed test features into model
    outputs = model(features)
    
    # Loss and optimization
    loss = criterion(outputs, targets)
    
    # Get test predictions and number of correct predictions
    _, predictions = torch.max(outputs.data, 1) 
    correct = torch.sum(predictions == targets.data)
    
    corrects = predictions == targets.data
    
#     for i, cor in enumerate(corrects):
#         if predictions[i] == 2 and cor == 0 and test_targets[i] == 1:
#             print(test_filepaths[i])
#             print(test_targets[i])
#             img = test_features[i]
#             plt.imshow(img)
#             plt.show()
            
    
    image_indices = list(range(0, total))
#     plot_images(loss, image_indices, test_filepaths, targets, predictions, phase='test')
    
    test_accuracy = correct / total * 100  
    print('Test accuracy: {:.2f}%'.format(test_accuracy))
    show_cm(list(targets.data), list(predictions))
    
test_model(current_model_train, test_dataloader)<jupyter_output>Test accuracy: 68.01%
{'fraction_non_alphanumeric': 0.09687381223869251, 'fraction_numerical': 0.015678449258836945, 'mean_word_length': 2.660059120153017, 'pattern_counts': {'":': 0, '<': 64, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '617586', 'license': 'MIT', 'n_tokens_mistral': 6519, 'n_tokens_neox': 6100, 'n_words': 1466, 'path': 'notebooks/.ipynb_checkpoints/Resnet-18 Basic-checkpoint.ipynb', 'repo_name': 'TimMolleman/FogDetection', 'size': '21048'}