<jupyter_start><jupyter_text>**AUTHOR-<NAME>**\Importing Required Libraries<jupyter_code>import torch
import torch.nn as nn<jupyter_output><empty_output><jupyter_text>Creating class named self_Attention where we will take embeded sizes and heads of the input layers. In our paper the embaded size is dk=dv = 512/8 =64.And the heads is 8<jupyter_code>class self_attention(nn.Module):
    def __init__(self, embedded_size, heads):
        super(self_attention, self).__init__()
        self.embedded_size = embedded_size
        self.heads = heads
        self.head_dimension = embedded_size // heads

        assert (self.head_dimension * heads == embedded_size), "embedding size needs to be divisible by heads"

        self.values = nn.Linear(self.head_dimension, self.head_dimension, bias=False)
        self.keys = nn.Linear(self.head_dimension, self.head_dimension, bias=False)
        self.queries = nn.Linear(self.head_dimension, self.head_dimension, bias=False)
        self.afc_out = nn.Linear(heads * self.head_dimension, embedded_size)
#Now we want to get the number of training examples. In paper training examples are English sentences where it has to get broken into saveral number
#of subwords which we never know. So the Query, Value and Key length will be depending on where we use the ATTENTION MECHANISM. So overall it 
#will depend on source sentence length and target sentence length.
    def forward(self, values, keys, query, mask):
        # Get number of training examples
        N = query.shape[0]

        value_len, key_len, query_len = values.shape[1], keys.shape[1], query.shape[1]

        # Split the embedding into self.heads different pieces
        values = values.reshape(N, value_len, self.heads, self.head_dimension)
        keys = keys.reshape(N, key_len, self.heads, self.head_dimension)
        query = query.reshape(N, query_len, self.heads, self.head_dimension)

        values = self.values(values)  
        keys = self.keys(keys)  
        queries = self.queries(query)  



        energy = torch.einsum("nqhd,nkhd->nhqk", [queries, keys])

        # Mask padded indices so their weights become 0
        if mask is not None:
            energy = energy.masked_fill(mask == 0, float("-1e20"))
        attention = torch.softmax(energy / (self.embedded_size ** (1 / 2)), dim=3)
        # attention shape: (N, heads, query_len, key_len)

        out = torch.einsum("nhql,nlhd->nqhd", [attention, values]).reshape(N, query_len, self.heads * self.head_dimension)

        out = self.afc_out(out)   
        return out<jupyter_output><empty_output><jupyter_text>**Transformers Block construction**<jupyter_code>class transblock(nn.Module):
    def __init__(self, embedded_size, heads, dropout, forward_expansion):
        super(transblock, self).__init__()
        self.attention = self_attention(embedded_size, heads)
        self.norm1 = nn.LayerNorm(embedded_size)
        self.norm2 = nn.LayerNorm(embedded_size)

        self.feed_forward = nn.Sequential(
            nn.Linear(embedded_size, forward_expansion * embedded_size),
            nn.ReLU(),
            nn.Linear(forward_expansion * embedded_size, embedded_size),
        )

        self.dropout = nn.Dropout(dropout)

    def forward(self, value, key, query, mask):
        attention = self.attention(value, key, query, mask)
        x = self.dropout(self.norm1(attention + query))
        forward = self.feed_forward(x)
        out = self.dropout(self.norm2(forward + x))
        return out<jupyter_output><empty_output><jupyter_text>**Creating Encoder**<jupyter_code>class Encoder(nn.Module):
    def __init__(
        self,
        source_vocab_size,
        embedded_size,
        no_of_layers,
        heads,
        device,
        forward_expansion,
        dropout,
        max_length,
    ):

        super(Encoder, self).__init__()
        self.embedded_size = embedded_size
        self.device = device
        self.word_embedding = nn.Embedding(source_vocab_size, embedded_size)
        self.position_embedding = nn.Embedding(max_length, embedded_size)

        self.layers = nn.ModuleList(
            [
                transblock(
                    embedded_size,
                    heads,
                    dropout=dropout,
                    forward_expansion=forward_expansion,
                )
                for _ in range(no_of_layers)
            ]
        )

        self.dropout = nn.Dropout(dropout)

    def forward(self, x, mask):
        N, seq_length = x.shape
        positions = torch.arange(0, seq_length).expand(N, seq_length).to(self.device)
        out = self.dropout(
            (self.word_embedding(x) + self.position_embedding(positions))
        )
        for layer in self.layers:
            out = layer(out, out, out, mask)

        return out<jupyter_output><empty_output><jupyter_text>**Creating Decoder Block**<jupyter_code>class de_block(nn.Module):
    def __init__(self, embedded_size, heads, forward_expansion, dropout, device):
        super(de_block, self).__init__()
        self.norm = nn.LayerNorm(embedded_size)
        self.attention = self_attention(embedded_size, heads=heads)
        self.transformer_block = transblock(
            embedded_size, heads, dropout, forward_expansion
        )
        self.dropout = nn.Dropout(dropout)

    def forward(self, x, value, key, source_mask, target_mask):
        attention = self.attention(x, x, x, target_mask)
        query = self.dropout(self.norm(attention + x))
        out = self.transformer_block(value, key, query, source_mask)
        return out<jupyter_output><empty_output><jupyter_text>Creating Decoder<jupyter_code>class Decoder(nn.Module):
    def __init__(
        self,
        target_vocab_size,
        embedded_size,
        no_of_layers,
        heads,
        forward_expansion,
        dropout,
        device,
        max_length,
    ):
        super(Decoder, self).__init__()
        self.device = device
        self.word_embedding = nn.Embedding(target_vocab_size, embedded_size)
        self.position_embedding = nn.Embedding(max_length, embedded_size)

        self.layers = nn.ModuleList(
            [
                de_block(embedded_size, heads, forward_expansion, dropout, device)
                for _ in range(no_of_layers)
            ]
        )
        self.afc_out = nn.Linear(embedded_size, target_vocab_size)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x, enc_out, source_mask, target_mask):
        N, seq_length = x.shape
        positions = torch.arange(0, seq_length).expand(N, seq_length).to(self.device)
        x = self.dropout((self.word_embedding(x) + self.position_embedding(positions)))

        for layer in self.layers:
            x = layer(x, enc_out, enc_out, source_mask, target_mask)

        out = self.afc_out(x)

        return out<jupyter_output><empty_output><jupyter_text>Now we are done with Encoder as well as decoder part so let's put them together.<jupyter_code>class Transformer(nn.Module):
    def __init__(
        self,
        source_vocab_size,
        target_vocab_size,
        source_pad_index,
        target_pad_index,
        embedded_size=512,
        no_of_layers=6,
        forward_expansion=4,
        heads=8,
        dropout=0,
        device="cpu",
        max_length=100,
    ):

        super(Transformer, self).__init__()

        self.encoder = Encoder(
            source_vocab_size,
            embedded_size,
            no_of_layers,
            heads,
            device,
            forward_expansion,
            dropout,
            max_length,
        )

        self.decoder = Decoder(
            target_vocab_size,
            embedded_size,
            no_of_layers,
            heads,
            forward_expansion,
            dropout,
            device,
            max_length,
        )

        self.source_pad_index = source_pad_index
        self.target_pad_index = target_pad_index
        self.device = device

    def make_source_mask(self, src):
        source_mask = (src != self.source_pad_index).unsqueeze(1).unsqueeze(2)
        # (N, 1, 1, source length)
        return source_mask.to(self.device)

    def make_trg_mask(self, Target):
        N, target_length = Target.shape
        target_mask = torch.tril(torch.ones((target_length, target_length))).expand(N, 1, target_length, target_length)

        return target_mask.to(self.device)

    def forward(self, src, Target):
        source_mask = self.make_source_mask(src)
        target_mask = self.make_trg_mask(Target)
        encoder_source = self.encoder(src, source_mask)
        out = self.decoder(Target, encoder_source, source_mask, target_mask)
        return out<jupyter_output><empty_output><jupyter_text>**Now we are done with the whole structure of encoder and decoder of attention mechanism discussed in the paper. So now let's check it with a small example.**<jupyter_code>if __name__ == "__main__":
    device = torch.device("cpu")
    print(device)

    x = torch.tensor([[1, 5, 6, 4, 3, 9, 5, 2, 0], [1, 8, 7, 3, 4, 5, 6, 7, 2]]).to(
        device
    )
    Target = torch.tensor([[1, 7, 4, 3, 5, 9, 2, 0], [1, 5, 6, 2, 4, 7, 6, 2]]).to(device)

    source_pad_index = 0
    target_pad_index = 0
    source_vocab_size = 10
    target_vocab_size = 10
    model = Transformer(source_vocab_size, target_vocab_size, source_pad_index, target_pad_index, device=device).to(
        device
    )
    out = model(x, Target[:, :-1])
    print(out.shape)<jupyter_output>cpu
torch.Size([2, 7, 10])
{'fraction_non_alphanumeric': 0.09902341699044419, 'fraction_numerical': 0.008925758689488606, 'mean_word_length': 2.6144212523719164, 'pattern_counts': {'":': 1, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '325953', 'license': 'MIT', 'n_tokens_mistral': 3098, 'n_tokens_neox': 2942, 'n_words': 655, 'path': 'Attention_is_all_you_need.ipynb', 'repo_name': 'Hariomsmit/Attention-is-all-you-need', 'size': '9524'}