<jupyter_start><jupyter_text>PR: Adding LogSoftmax layer to Darknet for Cross Entropy Loss<NAME> - 2018/4/24  0. Proposed Change; Setup Dataset is the fast.ai ImageNet sampleset. Jupyter kernel restarted between ImageNet learner runs due to model size.<jupyter_code>%matplotlib inline
%reload_ext autoreload
%autoreload 2
from pathlib import Path
from fastai.conv_learner import *
# from fastai.models import darknet<jupyter_output><empty_output><jupyter_text>Current Darknet, and Proposed Changes:**NOTE**: `from .layers import *` changed to **`from fastai.layers import *`**, preventing ModuleNotFoundError.<jupyter_code>import torch
import torch.nn as nn
import torch.nn.functional as F
from fastai.layers import *  ###  <<<------

class ConvBN(nn.Module):
    "convolutional layer then batchnorm"

    def __init__(self, ch_in, ch_out, kernel_size = 3, stride=1, padding=0):
        super().__init__()
        self.conv = nn.Conv2d(ch_in, ch_out, kernel_size=kernel_size, stride=stride, padding=padding, bias=False)
        self.bn = nn.BatchNorm2d(ch_out, momentum=0.01)
        self.relu = nn.LeakyReLU(0.1, inplace=True)

    def forward(self, x): return self.relu(self.bn(self.conv(x)))

class DarknetBlock(nn.Module):
    def __init__(self, ch_in):
        super().__init__()
        ch_hid = ch_in//2
        self.conv1 = ConvBN(ch_in, ch_hid, kernel_size=1, stride=1, padding=0)
        self.conv2 = ConvBN(ch_hid, ch_in, kernel_size=3, stride=1, padding=1)

    def forward(self, x): return self.conv2(self.conv1(x)) + x

class Darknet(nn.Module):
    "Replicates the darknet classifier from the YOLOv3 paper (table 1)"

    def make_group_layer(self, ch_in, num_blocks, stride=1):
        layers = [ConvBN(ch_in,ch_in*2,stride=stride)]
        for i in range(num_blocks): layers.append(DarknetBlock(ch_in*2))
        return layers

    def __init__(self, num_blocks, num_classes=1000, start_nf=32):
        super().__init__()
        nf = start_nf
        layers = [ConvBN(3, nf, kernel_size=3, stride=1, padding=1)]
        for i,nb in enumerate(num_blocks):
            layers += self.make_group_layer(nf, nb, stride=(1 if i==1 else 2))
            nf *= 2
        layers += [nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(nf, num_classes)]
        self.layers = nn.Sequential(*layers)

    def forward(self, x): return self.layers(x)

########################
### Proposed Version ###
class PR_Darknet(nn.Module):
    "Replicates the darknet classifier from the YOLOv3 paper (table 1)"

    def make_group_layer(self, ch_in, num_blocks, stride=1):
        layers = [ConvBN(ch_in,ch_in*2,stride=stride)]
        for i in range(num_blocks): layers.append(DarknetBlock(ch_in*2))
        return layers

    def __init__(self, num_blocks, num_classes=1000, start_nf=32):
        super().__init__()
        nf = start_nf
        layers = [ConvBN(3, nf, kernel_size=3, stride=1, padding=1)]
        for i,nb in enumerate(num_blocks):
            layers += self.make_group_layer(nf, nb, stride=(1 if i==1 else 2))
            nf *= 2
        layers += [nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(nf, num_classes)]
        layers += [nn.LogSoftmax()] ###  <<<------
        self.layers = nn.Sequential(*layers)

    def forward(self, x): return self.layers(x)
### /Proposed Version ###
########################
    
def darknet_53(num_classes=1000):    return Darknet([1,2,8,8,4], num_classes)
def darknet_small(num_classes=1000): return Darknet([1,2,4,8,4], num_classes)
def darknet_mini(num_classes=1000): return Darknet([1,2,4,4,2], num_classes, start_nf=24)
def darknet_mini2(num_classes=1000): return Darknet([1,2,8,8,4], num_classes, start_nf=16)
def darknet_mini3(num_classes=1000): return Darknet([1,2,4,4], num_classes)

# demonstrator
def PR_darknet_53(num_classes=1000):    return PR_Darknet([1,2,8,8,4], num_classes)
def display_head(fastai_learner, λ_name=None, show_nums=False):
    """displays final conv block and network head."""
    # parse
    if λ_name == None: 
        λ_name='DarknetBlock'
        fastai_learner = fastai_learner[0]
    if show_nums:
        fastai_learner = str(fastai_learner).split('\n')
        n = len(fastai_learner)
    else:
        n = len(str(fastai_learner).split('\n'))
        
    j = 1
    # find final conv block
    for i in range(n):
        if λ_name in str(fastai_learner[-j]): break
        j += 1
    # print head & 'neck'
    for i in range(j):
        print(fastai_learner[i-j])
        
# don't mind the λ's.. l's alone look too much like 1's
# It's easy to switch keyboards on a Mac or Windows (ctrl/win-space)
# fn NOTE: the `learner[0]` for Darknet is the same as `learner`
#          for other models; hence the if/else logic to keep printouts neat
#          show_nums displays layer numbers - kinda
PATH = Path('data/imagenet')
sz = 256
bs = 32
tfms = tfms_from_stats(imagenet_stats, sz)
model_data = ImageClassifierData.from_paths(PATH, bs=bs, tfms=tfms, val_name='train')<jupyter_output><empty_output><jupyter_text>1. fastai Darknet Loss Function  Darknet without LogSoftmax layer and with NLL loss<jupyter_code>f_model = darknet_53()
learner = ConvLearner.from_model_data(f_model, model_data)<jupyter_output><empty_output><jupyter_text>fastai.conv_learner logic sets criterion to [torch.nn.functional.nll_loss](https://github.com/pytorch/pytorch/blob/master/torch/nn/functional.pyL1294-L1294):<jupyter_code>learner.crit<jupyter_output><empty_output><jupyter_text>There is no final activation layer. The criterion will be applied to the final layer's output:<jupyter_code>display_head(learner)<jupyter_output>DarknetBlock(
  (conv1): ConvBN(
    (conv): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
    (bn): BatchNorm2d(512, eps=1e-05, momentum=0.01, affine=True)
    (relu): LeakyReLU(0.1, inplace)
  )
  (conv2): ConvBN(
    (conv): Conv2d(512, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (bn): BatchNorm2d(1024, eps=1e-05, momentum=0.01, affine=True)
    (relu): LeakyReLU(0.1, inplace)
  )
)
AdaptiveAvgPool2d(output_size=1)
Flatten(
)
Linear(in_features=1024, out_features=1000, bias=True)<jupyter_text>In this case the Learning Rate Finder 'fails' due to very small - often negative - loss values.<jupyter_code>learner.lr_find()
learner.sched.plot()<jupyter_output><empty_output><jupyter_text>Sometimes the LR Finder manages to produce a plot, but the results leave much to be desired:<jupyter_code>learner.lr_find()
learner.sched.plot()<jupyter_output><empty_output><jupyter_text>Darknet with Cross Entropy loss<jupyter_code>f_model = darknet_53()
learner = ConvLearner.from_model_data(f_model, model_data, crit=F.cross_entropy)
learner.crit<jupyter_output><empty_output><jupyter_text>There is no final activation layer. The criterion will be applied to the final layer's output:<jupyter_code>display_head(learner)
learner.lr_find()
learner.sched.plot()<jupyter_output><empty_output><jupyter_text>This is the shape of plot we expect to see.  Proposal: Darknet with LogSoftmax layer and NLL loss (Cross Entropy loss)<jupyter_code>f_model = PR_darknet_53()
learner = ConvLearner.from_model_data(f_model, model_data)<jupyter_output><empty_output><jupyter_text>fastai.conv_learner logic sets criterion to [torch.nn.functional.nll_loss](https://github.com/pytorch/pytorch/blob/master/torch/nn/functional.pyL1294-L1294):<jupyter_code>learner.crit<jupyter_output><empty_output><jupyter_text>However [cross_entropy](https://github.com/pytorch/pytorch/blob/master/torch/nn/functional.pyL1408-L1442) *is* NLL(LogSoftmax). The final layer is a LogSoftmax activation. The NLL criterion applied to its output will produce a Cross Entropy loss function.<jupyter_code>display_head(learner)
learner.lr_find()
learner.sched.plot()<jupyter_output><empty_output><jupyter_text>2. Learner Comparison: ResNet18 & DarkNet53 In working on this, I found some behavior that seemed odd, but may be normal. The CIFAR-10 dataset from fast.ai will be used here.<jupyter_code>from fastai.conv_learner import *
PATH = Path('data/cifar10')
sz = 64 # darknet53 architecture can't handle 32x32 small input
bs = 64
tfms = tfms_from_stats(imagenet_stats, sz)
model_data = ImageClassifierData.from_paths(PATH, bs=bs, tfms=tfms, val_name='test')<jupyter_output><empty_output><jupyter_text>A version of darknet.py with the proposed changes above is used.<jupyter_code>from fastai.models import darknet<jupyter_output><empty_output><jupyter_text>Comparing resnet18 from PyTorch to resnet18 from FastAI:<jupyter_code>from torchvision.models import resnet18
resnet18<jupyter_output><empty_output><jupyter_text>The fastai library does not alter the resnet18 model it imports from PyTorch.For comparison, the darknet53 import from fastai looks like this:<jupyter_code>darknet.darknet_53<jupyter_output><empty_output><jupyter_text>By contrast, the types of the initialized models:<jupyter_code>type(resnet18(num_classes=10))
type(darknet.darknet_53(num_classes=10))<jupyter_output><empty_output><jupyter_text>The PyTorch ResNet18 model has no output activation layer.<jupyter_code>f_model = resnet18()
display_head(str(f_model), λ_name='BasicBlock', show_nums=True)<jupyter_output>(1): BasicBlock(
      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)
      (relu): ReLU(inplace)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)
    )
  )
  (avgpool): AvgPool2d(kernel_size=7, stride=7, padding=0, ceil_mode=False, count_include_pad=True)
  (fc): Linear(in_features=512, out_features=1000, bias=True)
)<jupyter_text>When a learner is intialized via `ConvLearner.pretrained`, the fastai library adds a classifier head to the model via the `ConvnetBuilder` class. **NOTE** that the definition of the model is passed in, and *not* a model object.<jupyter_code>learner = ConvLearner.pretrained(resnet18, model_data)
display_head(learner, λ_name='BasicBlock', show_nums=True)<jupyter_output>(1): BasicBlock(
      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)
      (relu): ReLU(inplace)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)
    )
  )
  (8): AdaptiveConcatPool2d(
    (ap): AdaptiveAvgPool2d(output_size=(1, 1))
    (mp): AdaptiveMaxPool2d(output_size=(1, 1))
  )
  (9): Flatten(
  )
  (10): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True)
  (11): Dropout(p=0.25)
  (12): Linear(in_features=1024, out_features=512, bias=True)
  (13): ReLU()
  (14): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True)
  (15): Dropout(p=0.5)
  (16): Linear(in_features=512, out_features=10, bias=True)
  (17): LogSoftmax()
)<jupyter_text>***the fastai library adds the necessary LogSoftmax layer to the end of the model*****NOTE**: default constructor for resnet18 & darknet is 1000 classes (ImageNet). fastai lib finds the correct `num_classes` from the `ModelData` object. That's why the resnet18 model above has 1000 output features, and the resnet18 learner below it has the correct 10. The criterion, the loss function, of the learner is still `F.nll_loss`:<jupyter_code>learner.crit<jupyter_output><empty_output><jupyter_text>But since the final layer is an `nn.LogSoftmax`, the effective loss function is Cross Entropy.**NOTE** that this does **not** happen when the learner is initalized via `.from_model_data`:<jupyter_code>learner = ConvLearner.from_model_data(resnet18(num_classes=10), model_data)
display_head(learner, λ_name='BasicBlock', show_nums=True)<jupyter_output>(1): BasicBlock(
      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)
      (relu): ReLU(inplace)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)
    )
  )
  (avgpool): AvgPool2d(kernel_size=7, stride=7, padding=0, ceil_mode=False, count_include_pad=True)
  (fc): Linear(in_features=512, out_features=10, bias=True)
)<jupyter_text>'Strange/Normal' behavior: `ConvLearner.pretrained` will only accept model definitions, **not** models themselves:<jupyter_code>learner = ConvLearner.pretrained(resnet18, model_data)
learner = ConvLearner.pretrained(resnet18(num_classes=10), model_data)<jupyter_output><empty_output><jupyter_text>However the current version of Darknet is not accepted by `ConvLearner.pretrained` at all. This makes sense, given that the model is not yet pretrained, but also suggests further work is needed to integrate the model into the library.<jupyter_code>learner = ConvLearner.pretrained(darknet.darknet_53, model_data)
learner = ConvLearner.pretrained(darknet.darknet_53(num_classes=10), model_data)<jupyter_output><empty_output><jupyter_text>The `from_model_data` method works, as seen in section 1. ---  Misc<jupyter_code># Use this version of `display_head` if the other is too finicky for you.
# NOTE: fastai learners other than darknet will have to be entered as:
#       [str(learner_or_model).split('\n')]

def display_head(fastai_learner, λ_name=None):
    """displays final conv block and network head."""
    n = len(fastai_learner[0])
    if λ_name == None: λ_name='DarknetBlock'
    j = 1
    # find final conv block
    for i in range(n):
        if λ_name in str(fastai_learner[0][-j]): break
        j += 1
    # print head & 'neck'
    for i in range(j):
        print(fastai_learner[0][i-j])

# display_head(learner, λ_name='BasicBlock')
display_head(learner1) #darknet learner
print('--------')
display_head([str(learner2).split('\n')], λ_name='BasicBlock') #resnet learner
print('--------')
display_head([str(f_model).split('\n')], λ_name='BasicBlock') #resnet model<jupyter_output>DarknetBlock(
  (conv1): ConvBN(
    (conv): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
    (bn): BatchNorm2d(512, eps=1e-05, momentum=0.01, affine=True)
    (relu): LeakyReLU(0.1, inplace)
  )
  (conv2): ConvBN(
    (conv): Conv2d(512, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (bn): BatchNorm2d(1024, eps=1e-05, momentum=0.01, affine=True)
    (relu): LeakyReLU(0.1, inplace)
  )
)
AdaptiveAvgPool2d(output_size=1)
Flatten(
)
Linear(in_features=1024, out_features=1000, bias=True)
--------
    (1): BasicBlock(
      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)
      (relu): ReLU(inplace)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)
    )
  )
  (8): AdaptiveConcatPool2d(
    (ap): AdaptiveAvgPool2d(output_size=([...]
{'fraction_non_alphanumeric': 0.13736154407375473, 'fraction_numerical': 0.04437222259070107, 'mean_word_length': 5.156798693344222, 'pattern_counts': {'":': 0, '<': 99, '<?xml version=': 0, '>': 93, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '24', 'dir': 'jupyter-structured-clean-dedup', 'id': '130300', 'license': 'MIT', 'n_tokens_mistral': 5904, 'n_tokens_neox': 5454, 'n_words': 1250, 'path': 'FADL2/darknet_loss_PR.ipynb', 'repo_name': 'WNoxchi/Kawkasos', 'size': '15083'}