<jupyter_start><jupyter_text>TabNet https://arxiv.org/pdf/1908.07442.pdf  Global architecture   Feature Transformer  Видно, что есть повторяющиеся последовательности слоев => выделим их в отдельный слой.  GLU Layer<jupyter_code>class GLULayer(nn.Module):

    def __init__(self, input_size, output_size):
        super(GLULayer, self).__init__()

        self.fc = torch.nn.Linear(input_size, output_size)
        self.fc_bn = torch.nn.BatchNorm1d(output_size)

    def init_weights(self, m):
        if type(m) == nn.Linear:
            torch.nn.init.xavier_uniform(m.weight)
            # m.bias.data.fill_(0.001)

    def forward(self, input_data):
        output = self.fc(input_data)
        output = self.fc_bn(output)
        output = torch.nn.functional.glu(output)

        return output<jupyter_output><empty_output><jupyter_text><jupyter_code>class FeatureTransformer(nn.Module):

    def __init__(self, nrof_glu, input_size, output_size):
        super(FeatureTransformer, self).__init__()

        self.scale = torch.sqrt(torch.FloatTensor([0.5]))

        self.nrof_glu = nrof_glu
        self.glu_layers = []

        for i in range(nrof_glu):
            self.glu_layers.append(GLULayer(input_size[i], output_size))

    def forward(self, input_data):
        layer_input_data = input_data
        for i in range(self.nrof_glu):
            layer_input_data = torch.add(layer_input_data, self.glu_layers[i](layer_input_data))
            layer_input_data = layer_input_data * self.scale<jupyter_output><empty_output><jupyter_text>Attentive Transformer<jupyter_code>class AttentiveTransformer(nn.Module):

    def __init__(self):
        super(AttentiveTransformer, self).__init__()


    def init_weights(self, m):
        if type(m) == nn.Linear:
            torch.nn.init.xavier_uniform(m.weight)
            # m.bias.data.fill_(0.001)

    def forward(self, input_data):

        return output<jupyter_output><empty_output><jupyter_text>Sparsemax https://github.com/KrisKorrel/sparsemax-pytorch/blob/master/sparsemax.py  Собираем все слои в одну большую сеть<jupyter_code>class TabNet(nn.Module):
    def __init__(self):
        super(TabNet, self).__init__()




    def init_weights(self, m):
        if type(m) == nn.Linear:
            torch.nn.init.xavier_uniform(m.weight)
            # m.bias.data.fill_(0.001)

    def forward(self, input_data):

        return<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11290322580645161, 'fraction_numerical': 0.009925558312655087, 'mean_word_length': 3.236427320490368, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '67816', 'license': 'MIT', 'n_tokens_mistral': 874, 'n_tokens_neox': 857, 'n_words': 128, 'path': 'week08_09/TabNet/TabNet.ipynb', 'repo_name': 'romankop/SiriusDL', 'size': '2418'}