<jupyter_start><jupyter_text>Helpers<jupyter_code>#https://www.kaggle.com/maxlenormand/cropping-to-character-resizing-images
def crop_resize_scaled(df, resize_size = 128):
    HEIGHT = 137
    WIDTH = 236
    CROP_SIZE = resize_size
    original_img_size = HEIGHT * WIDTH
    cropped_img_size = CROP_SIZE * CROP_SIZE
    
    print(f"Original shape of images: {original_img_size}\nCropped & resized shape of images: {cropped_img_size}")
    print(f"Reduction fatio: {np.round(original_img_size/cropped_img_size, 3)}")
    print(df.shape)
    resized_df = df.iloc[:, 1:].values.reshape(-1, HEIGHT, WIDTH)
    
    cropped_imgs = {}
    #for img_id in tqdm(range(df.shape[0])):
    for img_id in range(df.shape[0]):
        img = resized_df[img_id]
        _, thresh = cv2.threshold(img, 30, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
        contours, _ = cv2.findContours(thresh,cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)[-2:]
        
        idx = 0 
        ls_xmin = []
        ls_ymin = []
        ls_xmax = []
        ls_ymax = []
        for cnt in contours:
            idx += 1
            x,y,w,h = cv2.boundingRect(cnt)
            ls_xmin.append(x)
            ls_ymin.append(y)
            ls_xmax.append(x + w)
            ls_ymax.append(y + h)
        xmin = min(ls_xmin)
        ymin = min(ls_ymin)
        xmax = max(ls_xmax)
        ymax = max(ls_ymax)

        roi = img[ymin:ymax,xmin:xmax]
        resized_roi = cv2.resize(roi, (resize_size, resize_size))
        cropped_imgs[df.image_id[img_id]] = resized_roi.reshape(-1)
        
    resized = pd.DataFrame(cropped_imgs).T.reset_index()
    resized.columns = resized.columns.astype(str)
    resized.rename(columns={'index':'image_id'},inplace=True)
    return resized #out_df

def imagePreprocessing(crop_func, file_in):
    df = pd.read_parquet(file_in)
    #df.reset_index(inplace=True,drop = True)#
    print(df.shape)
    cropped_df = crop_func(df, resize_size = 128)
    
    del df
    gc.collect()
    
    return cropped_df
#https://www.kaggle.com/iafoss/grapheme-fast-ai-starter-lb-0-964
class Loss_combine(nn.Module):
    def __init__(self):
        super().__init__()
        
    def forward(self, input, target,reduction='mean'):
        x1,x2,x3 = input
        x1,x2,x3 = x1.float(),x2.float(),x3.float()
        y = target.long()
        return 0.5*F.cross_entropy(x1,y[:,0],reduction=reduction) + 0.25*F.cross_entropy(x2,y[:,1],reduction=reduction) + \
          0.25*F.cross_entropy(x3,y[:,2],reduction=reduction)

class Metric_idx(Callback):
    def __init__(self, idx, average='macro'):
        super().__init__()
        self.idx = idx
        self.n_classes = 0
        self.average = average
        self.cm = None
        self.eps = 1e-9
        
    def on_epoch_begin(self, **kwargs):
        self.tp = 0
        self.fp = 0
        self.cm = None
    
    def on_batch_end(self, last_output:Tensor, last_target:Tensor, **kwargs):
        last_output = last_output[self.idx]
        last_target = last_target[:,self.idx]
        preds = last_output.argmax(-1).view(-1).cpu()
        targs = last_target.long().cpu()
        
        if self.n_classes == 0:
            self.n_classes = last_output.shape[-1]
            self.x = torch.arange(0, self.n_classes)
        cm = ((preds==self.x[:, None]) & (targs==self.x[:, None, None])) \
          .sum(dim=2, dtype=torch.float32)
        if self.cm is None: self.cm =  cm
        else:               self.cm += cm

    def _weights(self, avg:str):
        if self.n_classes != 2 and avg == "binary":
            avg = self.average = "macro"
            warn("average=`binary` was selected for a non binary case. \
                 Value for average has now been set to `macro` instead.")
        if avg == "binary":
            if self.pos_label not in (0, 1):
                self.pos_label = 1
                warn("Invalid value for pos_label. It has now been set to 1.")
            if self.pos_label == 1: return Tensor([0,1])
            else: return Tensor([1,0])
        elif avg == "micro": return self.cm.sum(dim=0) / self.cm.sum()
        elif avg == "macro": return torch.ones((self.n_classes,)) / self.n_classes
        elif avg == "weighted": return self.cm.sum(dim=1) / self.cm.sum()
        
    def _recall(self):
        rec = torch.diag(self.cm) / (self.cm.sum(dim=1) + self.eps)
        if self.average is None: return rec
        else:
            if self.average == "micro": weights = self._weights(avg="weighted")
            else: weights = self._weights(avg=self.average)
            return (rec * weights).sum()
    
    def on_epoch_end(self, last_metrics, **kwargs): 
        return add_metrics(last_metrics, self._recall())
    
Metric_grapheme = partial(Metric_idx,0)
Metric_vowel = partial(Metric_idx,1)
Metric_consonant = partial(Metric_idx,2)

class Metric_tot(Callback):
    def __init__(self):
        super().__init__()
        self.grapheme = Metric_idx(0)
        self.vowel = Metric_idx(1)
        self.consonant = Metric_idx(2)
        
    def on_epoch_begin(self, **kwargs):
        self.grapheme.on_epoch_begin(**kwargs)
        self.vowel.on_epoch_begin(**kwargs)
        self.consonant.on_epoch_begin(**kwargs)
    
    def on_batch_end(self, last_output:Tensor, last_target:Tensor, **kwargs):
        self.grapheme.on_batch_end(last_output, last_target, **kwargs)
        self.vowel.on_batch_end(last_output, last_target, **kwargs)
        self.consonant.on_batch_end(last_output, last_target, **kwargs)
        
    def on_epoch_end(self, last_metrics, **kwargs): 
        return add_metrics(last_metrics, 0.5*self.grapheme._recall() +
                0.25*self.vowel._recall() + 0.25*self.consonant._recall())
#https://github.com/osmr/imgclsmob/blob/master/pytorch/pytorchcv/models/common.py

def conv1x1(in_channels,
            out_channels,
            stride=1,
            groups=1,
            bias=False):
    """
    Convolution 1x1 layer.
    Parameters:
    ----------
    in_channels : int
        Number of input channels.
    out_channels : int
        Number of output channels.
    stride : int or tuple/list of 2 int, default 1
        Strides of the convolution.
    groups : int, default 1
        Number of groups.
    bias : bool, default False
        Whether the layer uses a bias vector.
    """
    return nn.Conv2d(
        in_channels=in_channels,
        out_channels=out_channels,
        kernel_size=1,
        stride=stride,
        groups=groups,
        bias=bias)


class SEBlock(nn.Module):
    """
    Squeeze-and-Excitation block from 'Squeeze-and-Excitation Networks,' https://arxiv.org/abs/1709.01507.
    Parameters:
    ----------
    channels : int
        Number of channels.
    reduction : int, default 16
        Squeeze reduction value.
    round_mid : bool, default False
        Whether to round middle channel number (make divisible by 8).
    activation : function, or str, or nn.Module, default 'relu'
        Activation function after the first convolution.
    out_activation : function, or str, or nn.Module, default 'sigmoid'
        Activation function after the last convolution.
    """
    def __init__(self,
                 channels,
                 reduction=16,
                 round_mid=False,
                 mid_activation=(lambda: nn.ReLU(inplace=True)),
                 out_activation=(lambda: nn.Sigmoid())):
        super(SEBlock, self).__init__()
        mid_channels = channels // reduction if not round_mid else round_channels(float(channels) / reduction)

        self.pool = nn.AdaptiveAvgPool2d(output_size=1)
        self.conv1 = conv1x1(
            in_channels=channels,
            out_channels=mid_channels,
            bias=True)
        self.activ = nn.ReLU(inplace=True)
        
        self.conv2 = conv1x1(
            in_channels=mid_channels,
            out_channels=channels,
            bias=True)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        identity = x
        w = self.pool(x)
        w = self.conv1(w)
        w = self.activ(w)
        w = self.conv2(w)
        w = self.sigmoid(w)
        x = x * w
        x = x + identity
        return x
class SAI(nn.Module):
    ''' SelfAttention with Identity '''
    
    def __init__(self, nf):
        super(SAI, self).__init__()
        
        self.sa = PooledSelfAttention2d(nf)
        self.bn = nn.BatchNorm2d(nf)
        
        
    def forward(self, x):
        ident = x
        out = self.sa(x)
        out = ident + out
        
        return nn.ReLU(out)

class HeadBlock(nn.Module):
    
    
    
    def __init__(self, nf, nc):
        super(HeadBlock, self).__init__()
        
        self.head1 = create_head(nf, nc[0])
        self.head2 = create_head(nf, nc[1])
        self.head3 = create_head(nf, nc[2])
        
        
    def forward(self, x):
        x1 = self.head1(x)
        x2 = self.head2(x)
        x3 = self.head3(x)
        
        return x1, x2, x3
#https://github.com/lukemelas/EfficientNet-PyTorch/blob/master/efficientnet_pytorch/utils.py
class SwishImplementation(torch.autograd.Function):
    @staticmethod
    def forward(ctx, i):
        result = i * torch.sigmoid(i)
        ctx.save_for_backward(i)
        return result

    @staticmethod
    def backward(ctx, grad_output):
        i = ctx.saved_variables[0]
        sigmoid_i = torch.sigmoid(i)
        return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i)))


class MemoryEfficientSwish(nn.Module):
    #def forward(self, x):
        #return x * torch.sigmoid(x)
    def forward(self, x):
        return SwishImplementation.apply(x)
class HeadBlockEff(nn.Module):
    
    def __init__(self, nf, nc):
        super(HeadBlockEff, self).__init__()
        
        #self.head1 = create_head(nf, 1024, bn_final = True)
        self.at1 = SelfAttention(nf)
        self.rl1 = nn.LeakyReLU(0.1, inplace=True)
        self.lin1 = nn.Linear(nf, nc[0])
        #self.sw1 = MemoryEfficientSwish()
        
        #self.head2 = create_head(nf, 1024, bn_final = True)
        self.at2 = SelfAttention(nf)
        self.rl2 = nn.LeakyReLU(0.1, inplace=True)
        self.lin2 = nn.Linear(nf, nc[1])
        #self.sw2 = MemoryEfficientSwish()
        
        #self.head3 = create_head(nf, 1024, bn_final = True)
        self.at3 = SelfAttention(nf)
        self.rl3 = nn.LeakyReLU(0.1, inplace=True)
        self.lin3 = nn.Linear(nf, nc[2])
        #self.sw3 = MemoryEfficientSwish()
        
        
    def forward(self, x):
        
        #x1 = self.head1(x)
        x1 = self.at1(x)
        x1 = self.rl1(x1)
        x1 = self.lin1(x1)
        #x1 = self.sw1(x1) 
        
        #x2 = self.head2(x)
        x2 = self.at2(x)
        x2 = self.rl2(x2)
        x2 = self.lin2(x2) 
        #x2 = self.sw2(x2) 
        
        #x3 = self.head3(x)
        x3 = self.at3(x)
        x3 = self.rl3(x3)
        x3 = self.lin3(x3) 
        #x3 = self.sw3(x3) 
        
        
        return x1, x2, x3
class HeadBlock0(nn.Module):
    
    def __init__(self, nf, nc):
        super(HeadBlock0, self).__init__()
        
        self.head1 = create_head(nf, 1024, bn_final = True)
        self.at1 = SelfAttention(nf//4)
        self.rl1 = nn.LeakyReLU(0.1, inplace=True)
        self.lin1 = nn.Linear(1024, nc[0])
        
        self.head2 = create_head(nf, 1024, bn_final = True)
        self.at2 = SelfAttention(nf//4)
        self.rl2 = nn.LeakyReLU(0.1, inplace=True)
        self.lin2 = nn.Linear(1024, nc[1])
        
        self.head3 = create_head(nf, 1024, bn_final = True)
        self.at3 = SelfAttention(nf//4)
        self.rl3 = nn.LeakyReLU(0.1, inplace=True)
        self.lin3 = nn.Linear(1024, nc[2])
        
        
    def forward(self, x):
        
        x1 = self.head1(x)
        x1 = self.at1(x1)
        x1 = self.rl1(x1)
        x1 = self.lin1(x1) 
        
        x2 = self.head2(x)
        x2 = self.at2(x2)
        x2 = self.rl2(x2)
        x2 = self.lin2(x2) 
        
        x3 = self.head3(x)
        x3 = self.at3(x3)
        x3 = self.rl3(x3)
        x3 = self.lin3(x3) 
        
        
        return x1, x2, x3


class HeadBlock(nn.Module):
    
    def __init__(self, nf, nc):
        super(HeadBlock, self).__init__()
        
        self.se1 = SEBlock(nf//2)
        #self.at1 = SEBlock(nf//2)
        self.head1 = create_head(nf, nc[0])
        #self.sa1 = SelfAttention(nf//4)
        self.se2 = SEBlock(nf//2)
        #self.at2 = SEBlock(nf//2)
        self.head2 = create_head(nf, nc[1])
        #self.sa2 = SelfAttention(nf//4)
        self.se3 = SEBlock(nf//2)
        #self.at3 = SEBlock(nf//2)
        self.head3 = create_head(nf, nc[2])
        #self.sa3 = SelfAttention(nf//4)
        
        
    def forward(self, x):
        x1 = self.se1(x)
        ##x1 = self.at1(x1)
        x1 = self.head1(x1)
        
        x2 = self.se2(x)
        ##x2 = self.at2(x2)
        x2 = self.head2(x2)
        
        x3 = self.se3(x)
        ##x3 = self.at3(x3)
        x3 = self.head3(x3)
        
        return x1, x2, x3
#https://www.kaggle.com/melissarajaram/model-ensembling-and-transfer-learning
class PixelImageItemList(ImageList):
    
    def open(self,fn):
        img_pixel = self.inner_df.loc[self.inner_df['fn'] == int(fn[2:])].values[0,1:(SIZE*SIZE+1)] #.values[0,1:32333]#
        img_pixel = img_pixel.reshape(SIZE,SIZE) #.reshape(137,236)#
        return vision.Image((pil2tensor(img_pixel,np.float32).div_(255)-1).abs_())
# https://github.com/fastai/fastai/blob/master/fastai/basic_train.py#L370
def my_validate(model:nn.Module, dl:DataLoader, loss_func:OptLossFunc=None, cb_handler:Optional[CallbackHandler]=None,
             pbar:Optional[PBar]=None, average=False, n_batch:Optional[int]=None)->Iterator[Tuple[Union[Tensor,int],...]]:
    "Calculate `loss_func` of `model` on `dl` in evaluation mode."
    model.eval()
    with torch.no_grad():
        val_losses,nums = [],[]
        grapheme_root, vowel_diacritic, consonant_diacritic = torch.Tensor(),torch.Tensor(),torch.Tensor()# = [],[],[]
        if cb_handler: cb_handler.set_dl(dl)
        for xb,yb in progress_bar(dl, parent=pbar, leave=(pbar is not None)):
            if cb_handler: xb, yb = cb_handler.on_batch_begin(xb, yb, train=False)
            val_loss = loss_batch(model, xb, yb, loss_func, cb_handler=cb_handler)[0]
            #val_losses.append(val_loss)
            
            #print(val_loss[0].shape,val_loss[1].shape,val_loss[2].shape)
            #grapheme_root.append(val_loss[0])
            #vowel_diacritic.append(val_loss[1])
            #consonant_diacritic.append(val_loss[2])
            grapheme_root = torch.cat((grapheme_root,val_loss[0]))
            vowel_diacritic = torch.cat((vowel_diacritic,val_loss[1]))
            consonant_diacritic = torch.cat((consonant_diacritic,val_loss[2]))
            
            if not is_listy(yb): yb = [yb]
            nums.append(first_el(yb).shape[0])
            if cb_handler and cb_handler.on_batch_end(grapheme_root[-1]): break
            #if cb_handler and cb_handler.on_batch_end(val_losses[-1]): break
            if n_batch and (len(nums)>=n_batch): break
        nums = np.array(nums, dtype=np.float32)
        #if average: return (to_np(torch.stack(val_losses)) * nums).sum() / nums.sum()
        #else:       
        
        return grapheme_root, vowel_diacritic, consonant_diacritic<jupyter_output><empty_output><jupyter_text>Load and transform Data<jupyter_code>'''
%%time
#source = Path('/home/kaggle/bengaliai-cv19/input')

for i in range(4):
    file_in = source/('bengaliai-cv19/test_image_data_' + str(i)+'.parquet')
    print(file_in)
    if i==0:
        df_test = imagePreprocessing(crop_resize_scaled,file_in)
    else:
        df_test.append(imagePreprocessing(crop_resize_scaled,file_in))
    print(str(i) + ' Done')
    
df_test.reset_index(inplace=True,drop = True)    
df_test['fn'] = df_test.index
df_test.head()
'''<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>%%time
#models = ['export_mymod_10E_SAI_mix_100d_Inv_0pad_kf.pkl','export_mymod_10E_SAI_mix_50d_Inv_0pad.pkl']#'export_10E_SAI_mix.pkl','export_10E_SAI_mix_100e.pkl','export_10E_SAI_mix_100d.pkl',
#models = ['export_mymod_10E_SAI_mix_50d_Inv_kf'+str(i)+'.pkl' for i in range(4)]

#models =['export_mymod_123E_SAI_mix_50d_Inv_0pad_2cos.pkl']

models = ['export_mymod_10E_SAI_mix_100d_Inv_0pad_2cos.pkl',
          'export_mymod_10E_SE_mix_50d_Inv_head0_2cos.pkl',
          'export_mymod_10E_SAI_mix_50d_Inv_0pad_0cos.pkl',
          'export_mymod_123E_eff-b2.pkl',
          'export_mymod_10E_eff-b4.pkl',
          'export_mymod_123E_SAI_mix_50d_Inv_0pad_2cos.pkl']


path = Path('/kaggle/input/10e-sai-mix/')

output = pd.DataFrame()

for i in range(4):
    file_in = source/('bengaliai-cv19/test_image_data_' + str(i)+'.parquet')
    print(file_in)
    df_test = imagePreprocessing(crop_resize_scaled,file_in)
    df_test.reset_index(inplace=True,drop = True)    
    df_test['fn'] = df_test.index
    print(str(i) + ' Data loaded')

    piil = PixelImageItemList.from_df(df=df_test,path='.',cols='fn')

    for m in models:
        learn = load_learner(path, m, test=piil).to_fp32()

        grapheme_root, vowel_diacritic, consonant_diacritic = my_validate(learn.model, learn.dl(ds_type=DatasetType.Test))
        df_g = pd.DataFrame({'rn':df_test.index, 'row_id': df_test['image_id']+str('_grapheme_root'), 'target': grapheme_root.numpy().argmax(axis=1)})
        df_v = pd.DataFrame({'rn':df_test.index, 'row_id': df_test['image_id']+str('_vowel_diacritic'), 'target': vowel_diacritic.numpy().argmax(axis=1)})
        df_c = pd.DataFrame({'rn':df_test.index, 'row_id': df_test['image_id']+str('_consonant_diacritic'), 'target': consonant_diacritic.numpy().argmax(axis=1)})

        #if i==0:
        #    output = (df_g.append(df_v).append(df_c)).sort_values(by=['rn','row_id'])
        #else:
        output = output.append((df_g.append(df_v).append(df_c)).sort_values(by=['rn','row_id']))

        del df_g
        del df_c
        del df_v
        del learn
        del grapheme_root
        del vowel_diacritic
        del consonant_diacritic

        gc.collect
    
    del piil
    gc.collect


#output.groupby('row_id')['target'].agg(pd.Series.mode).to_frame().sort_values('row_id').reset_index()#['target'].min
#pd.options.display.max_rows=100
output.head(20) #.shape
## Majority vote

output = output.groupby('row_id')['target'].agg(lambda x: pd.Series.mode(x)[0]).to_frame().sort_values('row_id').reset_index()

output.head()
#output.reset_index(inplace=True, drop = True)
    
output[['row_id','target']].to_csv('submission.csv', index=False)
output.head(10)
#piil = PixelImageItemList.from_df(df=df_test,path='.',cols='fn')
#piil
#path = Path('/kaggle/input/10e-sai-mix/')

#learn = load_learner(path, 'export_10E_SAI_mix.pkl',test=piil).to_fp32()#.to_fp16()
#grapheme_root, vowel_diacritic, consonant_diacritic = my_validate(learn.model, learn.dl(ds_type=DatasetType.Test))
#grapheme_root.shape, vowel_diacritic.shape, consonant_diacritic.shape
#for i in range(3):
#    print(grapheme_root[i].numpy().argmax(), vowel_diacritic[i].numpy().argmax(), consonant_diacritic[i].numpy().argmax())
###grapheme_root.numpy().argmax(axis=1),vowel_diacritic.numpy().argmax(axis=1),consonant_diacritic.numpy().argmax(axis=1)
#df_g = pd.DataFrame({'row_id': df_test['image_id']+str('_grapheme_root'), 'target': grapheme_root.numpy().argmax(axis=1)})
#df_g
#df_v = pd.DataFrame({'row_id': df_test['image_id']+str('_vowel_diacritic'), 'target': vowel_diacritic.numpy().argmax(axis=1)})
#df_v
#df_c = pd.DataFrame({'row_id': df_test['image_id']+str('_consonant_diacritic'), 'target': consonant_diacritic.numpy().argmax(axis=1)})
#df_c
#output = df_g.append(df_v).append(df_c).sort_values(by='row_id')
#output.head(10)
#output.to_csv('submission.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1390188679245283, 'fraction_numerical': 0.02631446540880503, 'mean_word_length': 2.5741773062398847, 'pattern_counts': {'":': 6, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 7, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '296494', 'license': 'Apache-2.0', 'n_tokens_mistral': 7683, 'n_tokens_neox': 7400, 'n_words': 1164, 'path': 'inference/bengali-cv19-inference-major-vote.ipynb', 'repo_name': 'joatom/kaggle-2020-bengali.ai', 'size': '19875'}