<jupyter_start><jupyter_text>TrainingThis notebook contains the image processing, architectures and training configuration for the five models I used.<jupyter_code>from fastai.vision import *
import pandas as pd
from tqdm import tqdm
import matplotlib.pyplot as plt
import numpy as np
import pretrainedmodels
import torchvision.models as torch_models

import dataprep
import model_utils_ext as m_util_x
import model_utils as m_util

from efficientnet_pytorch import EfficientNet

import cv2
source = Path('/home/kaggle/bengaliai-cv19/input')

bs = 64 # 64 for Model 3 else 128
EPOCHS_1 = 1 # for submittion model: 10-120
EPOCHS_2 = 1 #  for submittion model: 3-10<jupyter_output><empty_output><jupyter_text>Read the labels<jupyter_code>df_label = dataprep.load_labels()
df_label.head()
cls_count=[df_label.grapheme_root.nunique(), df_label.vowel_diacritic.nunique(), df_label.consonant_diacritic.nunique()]

print('Number of classes:', cls_count)<jupyter_output>Number of classes: [168, 11, 7]<jupyter_text>Load images and create databunch<jupyter_code>%%time
data = dataprep.data(bs)
data.show_batch(rows=3, figsize=(8,6))<jupyter_output><empty_output><jupyter_text>Model 1Modified resnext50<jupyter_code>arch = torch_models.resnext50_32x4d
def threeHeads(nf, nc):
    layers =[m_util.HeadBlock(nf,nc)]
    return nn.Sequential(*layers)
learn = cnn_learner(data, arch, cut=-2, custom_head=threeHeads(4096, cls_count), 
                    loss_func = m_util_x.Loss_combine(), 
                    metrics=[m_util_x.Metric_grapheme(),m_util_x.Metric_vowel(),m_util_x.Metric_consonant(),m_util_x.Metric_tot()]).to_fp16()

learn.clip_grad = 1.0
learn.unfreeze()<jupyter_output><empty_output><jupyter_text>Add self-attention layer after resnext blocks:<jupyter_code>for i in range(3):    
    learn.model[0][4][i].add_module('self_att1',m_util.SAI(256)) #PooledSelfAttention2d

for i in range(4):    
    learn.model[0][5][i].add_module('self_att2',m_util.SAI(512)) #PooledSelfAttention2d

for i in range(6): #23   #6 
    learn.model[0][6][i].add_module('self_att3',m_util.SAI(512)) #PooledSelfAttention2d

for i in range(3):    
    learn.model[0][7][i].add_module('self_att4',m_util.SAI(512)) #PooledSelfAttention2d

learn.lr_find()
learn.recorder.plot(suggestion=True)
lr = 1e-2
n_cycle = EPOCHS_1 # for competition: 10
csvlogger = callbacks.CSVLogger(learn,'learn_log_1_2')

reduceLR = callbacks.ReduceLROnPlateauCallback(learn=learn, monitor = 'valid_loss', mode = 'auto', patience = 2, factor = 0.2, min_delta = 0)

learn.fit_one_cycle(n_cycle, slice(lr), callbacks=[reduceLR, m_util_x.MixUpCallback(learn), csvlogger],pct_start=0.1)

#learn.save('stage1_mod1')
#learn.load('stage1_mod1')
learn.lr_find()
learn.recorder.plot_lr()
lr = 1e-3 
n_cycle = EPOCHS_2 # for competition: 10
csvlogger = callbacks.CSVLogger(learn,'learn_log_1_3')

reduceLR = callbacks.ReduceLROnPlateauCallback(learn=learn, monitor = 'valid_loss', mode = 'auto', patience = 2, factor = 0.2, min_delta = 0)
learn.fit_one_cycle(n_cycle, slice(lr), callbacks=[reduceLR, m_util_x.MixUpCallback(learn),csvlogger],pct_start=0.1)
#learn.recorder.plot_losses()
#learn.recorder.plot_lr()
#learn.recorder.plot_metrics()
#learn.save('mymod_160E_SAI_mix_50d_Inv_0pad_2cos')
#learn.export('export_mymod_160E_SAI_mix_50d_Inv_0pad_2cos.pkl')<jupyter_output><empty_output><jupyter_text>Model 2SE-Resnext 50<jupyter_code>def threeHeads(nf, nc):
    layers =[m_util.HeadBlock0(nf,nc)]
    return nn.Sequential(*layers)
learn = cnn_learner(data, m_util.se_resnext50, cut=-2, custom_head=threeHeads(4096, cls_count),  
                    loss_func = m_util_x.Loss_combine(), 
                    metrics=[m_util_x.Metric_grapheme(),m_util_x.Metric_vowel(),m_util_x.Metric_consonant(),m_util_x.Metric_tot()]).to_fp16()

learn.clip_grad = 1.0
learn.unfreeze()
lr = 1e-2
n_cycle = EPOCHS_1 # for competition: 10

csvlogger = callbacks.CSVLogger(learn,'learn_log_2_2')

reduceLR = callbacks.ReduceLROnPlateauCallback(learn=learn, monitor = 'valid_loss', mode = 'auto', patience = 2, factor = 0.2, min_delta = 0)
learn.fit_one_cycle(n_cycle, slice(lr), callbacks=[reduceLR, m_util_x.MixUpCallback(learn), csvlogger])

lr = 1e-3
n_cycle = EPOCHS_2 # for competition: 10

csvlogger = callbacks.CSVLogger(learn,'learn_log_2_3')

reduceLR = callbacks.ReduceLROnPlateauCallback(learn=learn, monitor = 'valid_loss', mode = 'auto', patience = 2, factor = 0.2, min_delta = 0)
learn.fit_one_cycle(n_cycle, slice(lr), callbacks=[reduceLR, m_util_x.MixUpCallback(learn), csvlogger])<jupyter_output><empty_output><jupyter_text>Model 3Modified Resnext-100<jupyter_code>arch = torch_models.resnext101_32x8d

def threeHeads(nf, nc):
    layers =[m_util.HeadBlock0(nf,nc)]
    return nn.Sequential(*layers)
learn = cnn_learner(data, arch, cut=-2, custom_head=threeHeads(4096, cls_count), 
                    loss_func = m_util_x.Loss_combine(), 
                    metrics=[m_util_x.Metric_grapheme(),m_util_x.Metric_vowel(),m_util_x.Metric_consonant(),m_util_x.Metric_tot()]).to_fp16()

learn.clip_grad = 1.0
learn.unfreeze()
for i in range(3):    
    learn.model[0][4][i].add_module('self_att1',m_util.SAI(256)) #PooledSelfAttention2d

for i in range(4):    
    learn.model[0][5][i].add_module('self_att2',m_util.SAI(512)) #PooledSelfAttention2d

for i in range(23): #23   #6 
    learn.model[0][6][i].add_module('self_att3',m_util.SAI(512)) #PooledSelfAttention2d

for i in range(3):    
    learn.model[0][7][i].add_module('self_att4',m_util.SAI(512)) #PooledSelfAttention2d

lr = 1e-2
n_cycle = EPOCHS_1 # for competition: 10

csvlogger = callbacks.CSVLogger(learn,'learn_log_3_2')

reduceLR = callbacks.ReduceLROnPlateauCallback(learn=learn, monitor = 'valid_loss', mode = 'auto', patience = 2, factor = 0.2, min_delta = 0)
learn.fit_one_cycle(n_cycle, slice(lr), callbacks=[reduceLR, m_util_x.MixUpCallback(learn), csvlogger])

lr = 1e-3
n_cycle = EPOCHS_2 # for competition: 10

csvlogger = callbacks.CSVLogger(learn,'learn_log_3_3')

reduceLR = callbacks.ReduceLROnPlateauCallback(learn=learn, monitor = 'valid_loss', mode = 'auto', patience = 2, factor = 0.2, min_delta = 0)
learn.fit_one_cycle(n_cycle, slice(lr), callbacks=[reduceLR, m_util_x.MixUpCallback(learn), csvlogger])<jupyter_output><empty_output><jupyter_text>Model 4EfficientNet-b2<jupyter_code>def threeHeads(nf, nc):
    layers =[m_util.HeadBlockEff(nf,nc)]
    return nn.Sequential(*layers)
eff_model = EfficientNet.from_pretrained('efficientnet-b2')
eff_model._fc=threeHeads(1408, cls_count)
learn = Learner(data, eff_model, 
                loss_func = m_util_x.Loss_combine(), 
                metrics=[m_util_x.Metric_grapheme(),m_util_x.Metric_vowel(),m_util_x.Metric_consonant(),m_util_x.Metric_tot()]).to_fp16()

learn.clip_grad = 1.0
learn.unfreeze()
lr = 1e-2
n_cycle = EPOCHS_1 # for competition: 120 

csvlogger = callbacks.CSVLogger(learn,'learn_log_4_2')

reduceLR = callbacks.ReduceLROnPlateauCallback(learn=learn, monitor = 'valid_loss', mode = 'auto', patience = 2, factor = 0.2, min_delta = 0)
learn.fit_one_cycle(n_cycle, slice(lr), callbacks=[reduceLR, m_util_x.MixUpCallback(learn), csvlogger])

lr = 3.63e-3
n_cycle = EPOCHS_2 # for competition: 3

csvlogger = callbacks.CSVLogger(learn,'learn_log_4_2')

reduceLR = callbacks.ReduceLROnPlateauCallback(learn=learn, monitor = 'valid_loss', mode = 'auto', patience = 2, factor = 0.2, min_delta = 0)
learn.fit_one_cycle(n_cycle, slice(lr), callbacks=[reduceLR, csvlogger],pct_start=0.1)<jupyter_output><empty_output><jupyter_text>Model 5EfficientNet-b4<jupyter_code>def threeHeads(nf, nc):
    layers =[m_util.HeadBlockEff(nf,nc)]
    return nn.Sequential(*layers)
eff_model = EfficientNet.from_pretrained('efficientnet-b4')
eff_model._fc=threeHeads(1792, cls_count)
lr = 1e-2
n_cycle = EPOCHS_1 # for competition: 10

csvlogger = callbacks.CSVLogger(learn,'learn_log_5_2')

reduceLR = callbacks.ReduceLROnPlateauCallback(learn=learn, monitor = 'valid_loss', mode = 'auto', patience = 2, factor = 0.2, min_delta = 0)
learn.fit_one_cycle(n_cycle, slice(lr), callbacks=[reduceLR, m_util_x.MixUpCallback(learn), csvlogger],pct_start=0.1)

lr = 1e-3
n_cycle = EPOCHS_2 # for competition: 10

csvlogger = callbacks.CSVLogger(learn,'learn_log_5_2')

reduceLR = callbacks.ReduceLROnPlateauCallback(learn=learn, monitor = 'valid_loss', mode = 'auto', patience = 2, factor = 0.2, min_delta = 0)
learn.fit_one_cycle(n_cycle, slice(lr), callbacks=[reduceLR, csvlogger])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13264101958933208, 'fraction_numerical': 0.03670049563370309, 'mean_word_length': 6.414698162729659, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '342462', 'license': 'Apache-2.0', 'n_tokens_mistral': 3491, 'n_tokens_neox': 3286, 'n_words': 521, 'path': 'training/Training.ipynb', 'repo_name': 'joatom/kaggle-2020-bengali.ai', 'size': '8474'}