<jupyter_start><jupyter_text>Define multi class model<jupyter_code>def get_y_fn(x):
  y = str(x).split(".")[0].split('_')[-2]
  return y

FOLD = 0
val_index = train_folds[FOLD]
FOLD = str(FOLD)
val_index
train_folds[0].shape<jupyter_output><empty_output><jupyter_text>define mixup<jupyter_code>!git clone https://github.com/nestordemeure/ManifoldMixupV2.git
%run /content/ManifoldMixupV2/manifold_mixup.py
OutputMixup()<jupyter_output><empty_output><jupyter_text>Define db and dls<jupyter_code>!git clone https://NadyaStrogankova:c12ca00be6ebdcb705be6f0e9fac559a684c3d3b@github.com/NadyaStrogankova/RainforestAudioKaggle.git
%run /content/RainforestAudioKaggle/transforms.py
cfg = AudioConfig.BasicMelSpectrogram(
        mel=True,
  #      to_db = False
        f_min=df["f_min"].min(),
        f_max=df["f_max"].max(),
        # n_fft=1024,
     #   n_mels=384,
     #   hop_length=292, # for size 768
       # n_fft = 892, 
        hop_length=748, 
        n_mels = 300,
      #  normalized=True
     #   power=3 # неплохо, возможно стоит дошибить рандомный power
   )

item_tfms = [Resample(28000),
             ResizeSignal(8000, pad_mode=AudioPadType.Repeat), 
            AddNoise(noise_level=0.05, color=NoiseColor.Pink),
            AddNoise(noise_level=0.05, color=NoiseColor.White),
           ]
batch_tfms = [ AudioToSpec.from_cfg(cfg), 
       #       Normalize_channel(),
       #       PowerSpec(), # увеличение контрастности
       #       TAmplitudeToDB(),
              Normalize_channel_1(),
    #          PowerSpec(1.5, 0.7),
              WhiteNoise(0.01, cfg),
              PinkNoise(0.01, cfg),
       #       RowNoise(0.025, cfg),
              LowerUpperFreq(cfg),
              Normalize_channel_2(),
              PowerSpec(2, 0.7), # увеличение контрастности
      #        Normalize_channel_3(),
              SGRoll(),
              Mask_via_torch(),
         #     Mono2Color()
         #     Normalize(ch_mean, ch_std, axes=(0, 1, 3)),
              ]
AddNoise.split_idx = 0
Mask.split_idx = 0
Mask_via_torch.split_idx = 0

SGRoll.split_idx = 0

auds = DataBlock(blocks = (AudioBlock, CategoryBlock),  
                 get_items=get_audio_files,
                 item_tfms = item_tfms,
                 #splitter = IndexSplitter(val_index.tolist()), #report unnesesary transform to list
                 splitter = RandomSplitter(),
                 get_y=get_y_fn,
                 batch_tfms = batch_tfms
                                  )
print(auds.summary(TRAIN_AUDIO_ROOT))
dls = auds.dataloaders(TRAIN_AUDIO_ROOT, bs=16, verbose=True, before_batch = batch_tfms, 
                       num_workers=2
                       #, shuffle=True
                      )

def _one_sample_positive_class_precisions(scores, truth):
    num_classes = scores.shape[0]
    pos_class_indices = np.flatnonzero(truth > 0)

    if not len(pos_class_indices):
        return pos_class_indices, np.zeros(0)

    retrieved_classes = np.argsort(scores)[::-1]

    class_rankings = np.zeros(num_classes, dtype=np.int)
    class_rankings[retrieved_classes] = range(num_classes)

    retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
    retrieved_class_true[class_rankings[pos_class_indices]] = True

    retrieved_cumulative_hits = np.cumsum(retrieved_class_true)

    precision_at_hits = (
            retrieved_cumulative_hits[class_rankings[pos_class_indices]] /
            (1 + class_rankings[pos_class_indices].astype(np.float)))
    return pos_class_indices, precision_at_hits

def lwlrap(scores, truth):
    #print(truth.shape, scores.shape)
    num_samples, num_classes = scores.shape
    scores = scores.cpu().numpy()
    gt = np.zeros((num_samples, num_classes))
    for n, i in enumerate(truth.cpu().numpy().astype(int)):
      gt[n, i] = 1
    assert gt.shape == scores.shape
    
    precisions_for_samples_by_classes = np.zeros((num_samples, num_classes))
    for sample_num in range(num_samples):
        pos_class_indices, precision_at_hits = _one_sample_positive_class_precisions(scores[sample_num, :], gt[sample_num, :])
        precisions_for_samples_by_classes[sample_num, pos_class_indices] = precision_at_hits

    labels_per_class = np.sum(gt > 0, axis=0)
    weight_per_class = labels_per_class / float(np.sum(labels_per_class))

    per_class_lwlrap = (np.sum(precisions_for_samples_by_classes, axis=0) /
                        np.maximum(1, labels_per_class))
    #return per_class_lwlrap, weight_per_class
    return (per_class_lwlrap * weight_per_class).sum()

from sklearn.metrics import label_ranking_average_precision_score 
def lwlrap_sklearn(scores, truth):
    #print(truth.shape, scores.shape, truth, scores)
    scores = scores.cpu().numpy()
    truth = truth.cpu().numpy()
    num_samples, num_classes = scores.shape
    gt = np.zeros((num_samples, num_classes))
    for n, i in enumerate(truth.astype(int)):
      gt[n, i] = 1
    assert gt.shape == scores.shape
    return label_ranking_average_precision_score(gt, scores)

from efficientnet_pytorch import EfficientNet
from timm import create_model
# from https://colab.research.google.com/github/muellerzr/Practical-Deep-Learning-for-Coders-2.0/blob/master/Computer%20Vision/05_EfficientNet_and_Custom_Weights.ipynb#scrollTo=VXPjDVUlJgCU
def init_layer(layer):
    nn.init.xavier_uniform_(layer.weight)

    if hasattr(layer, "bias"):
        if layer.bias is not None:
            layer.bias.data.fill_(0.)


def init_bn(bn):
    bn.bias.data.fill_(0.)
    bn.weight.data.fill_(1.0)


def interpolate(x: torch.Tensor, ratio: int):
    """Interpolate data in time domain. This is used to compensate the
    resolution reduction in downsampling of a CNN.

    Args:
      x: (batch_size, time_steps, classes_num)
      ratio: int, ratio to interpolate
    Returns:
      upsampled: (batch_size, time_steps * ratio, classes_num)
    """
    (batch_size, time_steps, classes_num) = x.shape
    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)
    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)
    return upsampled


def pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):
    """Pad framewise_output to the same length as input frames. The pad value
    is the same as the value of the last frame.
    Args:
      framewise_output: (batch_size, frames_num, classes_num)
      frames_num: int, number of frames to pad
    Outputs:
      output: (batch_size, frames_num, classes_num)
    """
    pad = framewise_output[:, -1:, :].repeat(
        1, frames_num - framewise_output.shape[1], 1)
    """tensor for padding"""

    output = torch.cat((framewise_output, pad), dim=1)
    """(batch_size, frames_num, classes_num)"""

    return output


class ConvBlock(nn.Module):
    def __init__(self, in_channels: int, out_channels: int):
        super().__init__()

        self.conv1 = nn.Conv2d(
            in_channels=in_channels,
            out_channels=out_channels,
            kernel_size=(3, 3),
            stride=(1, 1),
            padding=(1, 1),
            bias=False)

        self.conv2 = nn.Conv2d(
            in_channels=out_channels,
            out_channels=out_channels,
            kernel_size=(3, 3),
            stride=(1, 1),
            padding=(1, 1),
            bias=False)

        self.bn1 = nn.BatchNorm2d(out_channels)
        self.bn2 = nn.BatchNorm2d(out_channels)

        self.init_weight()

    def init_weight(self):
        init_layer(self.conv1)
        init_layer(self.conv2)
        init_bn(self.bn1)
        init_bn(self.bn2)

    def forward(self, input, pool_size=(2, 2), pool_type='avg'):

        x = input
        x = F.relu_(self.bn1(self.conv1(x)))
        x = F.relu_(self.bn2(self.conv2(x)))
        if pool_type == 'max':
            x = F.max_pool2d(x, kernel_size=pool_size)
        elif pool_type == 'avg':
            x = F.avg_pool2d(x, kernel_size=pool_size)
        elif pool_type == 'avg+max':
            x1 = F.avg_pool2d(x, kernel_size=pool_size)
            x2 = F.max_pool2d(x, kernel_size=pool_size)
            x = x1 + x2
        else:
            raise Exception('Incorrect argument!')

        return x


class AttBlock(nn.Module):
    def __init__(self,
                 in_features: int,
                 out_features: int,
                 activation="linear",
                 temperature=1.0):
        super().__init__()

        self.activation = activation
        self.temperature = temperature
        self.att = nn.Conv1d(
            in_channels=in_features,
            out_channels=out_features,
            kernel_size=1,
            stride=1,
            padding=0,
            bias=True)
        self.cla = nn.Conv1d(
            in_channels=in_features,
            out_channels=out_features,
            kernel_size=1,
            stride=1,
            padding=0,
            bias=True)

        self.bn_att = nn.BatchNorm1d(out_features)
        self.init_weights()
        self.in_channels=in_features
        self.out_channels=out_features

    def init_weights(self):
        init_layer(self.att)
        init_layer(self.cla)
        init_bn(self.bn_att)

    def forward(self, x):
        # x: (n_samples, n_in, n_time)
        #print(x.size())
        x = x.view(-1, self.in_channels, 1)
        
        #print(self.att(x).size())
        norm_att = torch.softmax(torch.clamp(self.att(x), -10, 10), dim=-1)
        cla = self.nonlinear_transform(self.cla(x))
        x = torch.sum(norm_att * cla, dim=2)

        #return x, norm_att, cla
        #print(x)
        return x

    def nonlinear_transform(self, x):
        if self.activation == 'linear':
            return x
        elif self.activation == 'sigmoid':
            return torch.sigmoid(x)
def create_timm_body(arch:str, pretrained=True, cut=None):
  model = create_model(arch, pretrained=pretrained, in_chans=1)
  if cut is None:
    ll = list(enumerate(model.children()))
    cut = next(i for i,o in reversed(ll) if has_pool_type(o))
  if isinstance(cut, int): return nn.Sequential(*list(model.children())[:cut])
  elif callable(cut): return cut(model)
  else: raise NamedError("cut must be either integer or function")
body = create_timm_body('tf_efficientnet_b3_ns', pretrained=True)

lin_ftrs=128
head = create_head(num_features_model(nn.Sequential(*body.children())) * (2), dls.c, ps=0.4, lin_ftrs=[lin_ftrs])
#head.__setitem__(4, nn.Linear(512, 256))
head.__setitem__(8, AttBlock(lin_ftrs, 24))
model = nn.Sequential(body, head)

apply_init(model[1], nn.init.kaiming_normal_)
len(model)
model
learn = Learner(dls,  model,
           # n_in=1, #<- Only audio specific modification here
        #    loss_func=LabelSmoothingCrossEntropyFlat(),
            cbs = OutputMixup(alpha=0.4),
            metrics=[accuracy, lwlrap],
            )
learn.to_fp16()
learn.loss_func
dls.show_batch(), dls.vocab
#learn.lr_find()
wdc wc  
0 fold: 
  0.799850	0.665927	0.841667	0.894023	
1 fold 1 step 
19	0.770710	0.800323	0.795833	0.872832
2 fold 1 step
17	0.871557	1.045797	0.766393	0.847755
3 fold 1 step
12	1.114541	0.701009	0.796681	0.876500	
4 fold 1 step 
16	0.896284	0.679707	0.828571	0.896921	
	
EXP_NAME = "b3_fold" + FOLD 
SUFFIX ="step1"
learn.freeze()
learn.fit(3, 1e-3, wd=1e-3)
learn.unfreeze()
learn.fit_one_cycle(25, 1e-3, wd=1e-3, div=8, div_final=5e2, pct_start=0.25,
                    cbs=[SaveModelCallback(fname = f'{EXP_NAME}_{SUFFIX}', with_opt=True)])
dls2 = auds.dataloaders(Path(DATA_ROOT/"samples_long"), bs=16, verbose=True, before_batch = batch_tfms, 
                       num_workers=2, splitter = IndexSplitter(val_index.tolist())
                      )
learn2 = Learner(dls2,  model,
           # n_in=1, #<- Only audio specific modification here
        #    loss_func=LabelSmoothingCrossEntropyFlat(),
            cbs = OutputMixup(alpha=0.4),
            metrics=[accuracy, lwlrap],
            ).to_fp16()

SUFFIX ="step2"
learn2.load(EXP_NAME+"_step1")
learn2.fit_one_cycle(10, slice(1e-3 / 5 / 4, 1e-3 / 5), wd=1e-3, div=8, div_final=1e3, pct_start=0.2,
                    cbs=[SaveModelCallback(fname = f'{EXP_NAME}_{SUFFIX}', with_opt=True)]
                    )<jupyter_output>Output mixup: the loss function is now properly wrapped.<jupyter_text>confusion matrix<jupyter_code>interp = ClassificationInterpretation.from_learner(learn2)
interp.plot_confusion_matrix()<jupyter_output><empty_output><jupyter_text>Predict all classes<jupyter_code>def new_act(x):
  return x

def tta2(learner, ds_idx=1, dl=None, n=4, item_tfms=None, batch_tfms=None, beta=0.25, use_max=False):
    "Return predictions on the `ds_idx` dataset or `dl` using Test Time Augmentation"
    if dl is None: dl = learner.dls[ds_idx]
    if item_tfms is not None or batch_tfms is not None: dl = dl.new(after_item=item_tfms, after_batch=batch_tfms)
    try:
        learner.recorder.before_epoch()
        with dl.dataset.set_split_idx(0), learner.no_mbar():
            if hasattr(learner,'progress'): learner.progress.mbar = master_bar(list(range(n)))
            aug_preds = []
            for i in learner.progress.mbar if hasattr(learner,'progress') else range(n):
                learner.epoch = i #To keep track of progress on mbar since the progress callback will use self.epoch
                aug_preds.append(learner.get_preds(dl=dl, inner=True, act=new_act)[0][None])
        aug_preds = torch.cat(aug_preds)
        aug_preds = aug_preds.max(0)[0] if use_max else aug_preds.mean(0)
        learner.epoch = n
        with dl.dataset.set_split_idx(1): preds,targs = learner.get_preds(dl=dl, inner=True,  act=new_act)
    finally: learner(event.after_fit)

    if use_max: return torch.stack([preds, aug_preds], 0).max(0)[0],targs
    preds = (aug_preds,preds) if beta is None else torch.lerp(aug_preds, preds, beta)
    return preds,targs
learn2.load("b3_fold"+FOLD+"_step2", with_opt=False, device=dls.device)
learn2.remove_cbs(OutputMixup)
learn2.cbs
#!ln -s drive/MyDrive/Colab\ Notebooks/RainForestAudio/data/ol_samples.tar.gz ol_samples.tar.gz  
# !ln -s drive/MyDrive/Colab\ Notebooks/RainForestAudio/data/val  val
#!ln -s drive/MyDrive/Colab\ Notebooks/RainForestAudio/data/fp_samples_long  fp_samples_long
#TEST_SAMPLES_AUDIO_ROOT = Path("/content/fp_samples_long")
TEST_SAMPLES_AUDIO_ROOT = Path("/content/samples")

#!ln -s drive/MyDrive/Colab\ Notebooks/RainForestAudio/data/val/ /content/val
VAL_AUDIO_ROOT = Path("/content/val")

test_ds = auds.new(VAL_AUDIO_ROOT)
test_dl = learn2.dls.test_dl(get_audio_files(TEST_SAMPLES_AUDIO_ROOT))
#print(test_dl.summary(TEST_TARGET_AUDIO_ROOT))
test_probas, *_ = tta2(learn2.to_fp32(), dl=test_dl, n=6, use_max=False, beta=1/7)
result=[]
for probas, fname in zip(test_probas, test_dl.items):
  print(fname)
  result.append([fname.stem.split("_")[0], fname.stem.split("_")[1], fname.stem, probas.numpy()])
res = pd.DataFrame(result, columns =["recording_id", "part_id", "fname", "probas"])
res["part_id"] = res["part_id"].astype(int)
res.to_csv("all_class_pred_"+FOLD+".csv", columns =["recording_id", "part_id", "fname", "probas"])
#res.to_pickle("fp_08_b3_"+FOLD+".pkl", protocol=4)
subm = []
for n, row in res.sort_values(by="recording_id").groupby(by="recording_id"):
  a = np.stack(row["probas"].values)
  pred = np.concatenate([np.array(n).reshape(1),
                         np.array(dls.vocab[np.argmax(a.max(axis=0))]).reshape(1),
                         #np.where(a>0.7, a, 0).sum(axis=0)
                         a.max(axis=0)
                         ])
  subm.append(pred)
  #print(n)
  if n == "047a7c4bf":
    #print(row["probas"].max(axis=1))
    cls = np.stack(row["probas"].values).max(axis=0).argsort()
    #print(np.stack(row["probas"].values).argsort())
    #print(dls.vocab[cls[-1]], dls.vocab[cls[-2]], dls.vocab[cls[-3]])
#print(subm)
submission = pd.DataFrame(subm, columns=["recording_id"] + ["top_cat"] + ["s" + i for i in dls.vocab])
submission.head(10)
submission.drop(["top_cat"], axis=1, inplace=True)
submission.to_csv("ss_08_b3_fold_"+FOLD+".csv", columns=["recording_id"] + ["s" + str(i) for i in range(24)], index=False)
from sklearn.metrics import accuracy_score
gt=[]
pred=[]
for n, row in submission.iterrows():
  try:
    gr_tr = df[df["recording_id"] == row["recording_id"]]["species_id"].to_numpy()
    if gr_tr.shape[0] > 1:
      if int(row["top_cat"]) in gr_tr:
        gt.append(int(row["top_cat"]))
      else:
        gt.append(gr_tr[0])     
    else: 
      gt.append(gr_tr[0])
    pred.append(int(row["top_cat"]))
  except:
    pass
accuracy_score(gt,pred), len(gt)
#gt, pred<jupyter_output><empty_output><jupyter_text>Усредненный сабмит<jupyter_code>subm = []
#subm.append(submission.sort_values(by="recording_id").to_numpy()[:,1:].astype(float))
for i in range(4):
  s = pd.read_csv("ss_08_b3_fold_"+str(i)+".csv")
  subm.append(s.sort_values(by="recording_id").to_numpy()[:,1:])
subm = np.stack(subm)
subm.shape
fs = pd.DataFrame(np.concatenate([submission.sort_values(by="recording_id").to_numpy()[:,0].reshape(-1,1), subm.sum(axis=0)], axis=1),  columns=s.columns)
fs.head()

fs.to_csv("subm_10_ssb3.csv", index=False)<jupyter_output><empty_output><jupyter_text>Усреднение предсказаний для fp отрезкови переименовать файлы для тренировки<jupyter_code>res1 = pd.read_pickle("/content/fp_08_b3_3.pkl")
res1
from sklearn.utils.extmath import softmax

def top_cat(x):
  val = softmax(x.values[1:25].reshape(1,-1).astype(np.float))
  #print(val.shape, val.argmax(axis=0))
  return val.argmax(), val.max() 



subm = []
#subm.append(submission.sort_values(by="recording_id").to_numpy()[:,1:].astype(float))
for i in range(5):
  s = pd.read_pickle("fp_08_b3_"+str(i)+".pkl")
  subm.append(s.to_numpy())
subm = np.concatenate(subm)
subm
fs = pd.DataFrame(subm, columns = s.columns)
fs = fs.groupby(by=["fname"])
result=[]
for row in fs:
  res = softmax(np.stack(row[1]["probas"].to_numpy()).sum(axis=0).reshape(1,-1))
  result.append([row[0], dls.vocab[res.argmax()], res.max()])
result

#fs = pd.DataFrame(np.concatenate([submission.sort_values(by="recording_id").to_numpy()[:,0].reshape(-1,1), subm.sum(axis=0)], axis=1),  columns=s.columns)
#fs["top_cat"] = fs.apply(lambda x: top_cat(x)[0], axis=1)
#fs["top_prob"] = fs.apply(lambda x: top_cat(x)[1], axis=1)
#fs.head()
SRC_PATH = Path("/content/fp_samples_long")
DEST_PATH= Path("/content/drive/MyDrive/Colab Notebooks/RainForestAudio/data/samples_mixed")
for row in result:
  if row[2] > 0.99:
    fname = "_".join([row[0].split("_")[0], row[1], row[0].split("_")[2]])
    print(fname)
    shutil.copy(SRC_PATH/(row[0]+".flac"), DEST_PATH/(fname+".flac"))
fs.to_csv("fp_08_b3.csv", index=False)<jupyter_output><empty_output><jupyter_text>Вспомогательное<jupyter_code>class MaskFreq(SpectrogramTransform):
    """Google SpecAugment frequency masking from https://arxiv.org/abs/1904.08779."""

    def __init__(self, num_masks=1, size=20, start=None, val=None):
        self.num_masks = num_masks
        self.size = size
        self.start = start
        self.val = val

    def encodes(self, sg: AudioSpectrogram) -> AudioSpectrogram:
        channel_mean = sg.contiguous().view(sg.size(0), -1).mean(-1)[:, None, None]
        mask_val = ifnone(self.val, channel_mean)
        if sg.ndim == 4:
          b, c, y, x = sg.shape
          # Position of the first mask
          start = ifnone(self.start, random.randint(0, y - self.size))
          for _ in range(self.num_masks):
              mask = torch.ones(self.size, x).cuda() * mask_val.cuda()
              mask = mask.view(b, c, self.size, x)
              #print("sg, mask:", sg.shape, mask.shape)
              if not 0 <= start <= y - self.size:
                  raise ValueError(
                      f"Start value '{start}' out of range for AudioSpectrogram of shape {sg.shape}"
                  )
              sg[:, :, start : start + self.size, :] = mask
              # Setting start position for next mask
              start = random.randint(0, y - self.size)
        else:
          c, y, x = sg.shape
          # Position of the first msk
          start = ifnone(self.start, random.randint(0, y - self.size))
          for _ in range(self.num_masks):
              mask = torch.ones(self.size, x) * mask_val
              if not 0 <= start <= y - self.size:
                  raise ValueError(
                      f"Start value '{start}' out of range for AudioSpectrogram of shape {sg.shape}"
                  )
              sg[:, start : start + self.size, :] = mask
              # Setting start position for next mask
              start = random.randint(0, y - self.size)
        return sg
cp b3_ss_09.tar drive/MyDrive/Colab\ Notebooks/RainForestAudio/<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13560059987421993, 'fraction_numerical': 0.027042716849692807, 'mean_word_length': 3.0845682671408814, 'pattern_counts': {'":': 1, '<': 38, '<?xml version=': 0, '>': 38, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '38631', 'license': 'MIT', 'n_tokens_mistral': 7850, 'n_tokens_neox': 7517, 'n_words': 1317, 'path': 'bag_of_notebooks/effnet_trainwfp_b3.ipynb', 'repo_name': 'NadyaStrogankova/RainforestAudioKaggle', 'size': '20671'}