def __init__(self, n_heads, n_layers, embedding_size, ffn_size, vocabulary_size, embedding=None, attention_dropout=0.0, relu_dropout=0.0, embeddings_scale=True, learn_positional_embeddings=False, padding_idx=None):
    super().__init__()
    self.embedding_size = embedding_size
    self.ffn_size = ffn_size
    self.n_layers = n_layers
    self.n_heads = n_heads
    self.dim = embedding_size
    self.embeddings_scale = embeddings_scale
    self.out_dim = embedding_size
    assert ((embedding_size % n_heads) == 0), 'Transformer embedding size must be a multiple of n_heads'
    n_positions = 1024
    self.embeddings = embedding
    self.position_embeddings = nn.Embedding(n_positions, embedding_size)
    if (not learn_positional_embeddings):
        create_position_codes(n_positions, embedding_size, out=self.position_embeddings.weight)
    else:
        nn.init.normal_(self.position_embeddings.weight, 0, (embedding_size ** (- 0.5)))
    self.layers = nn.ModuleList()
    for _ in range(self.n_layers):
        self.layers.append(TransformerDecoderLayer(n_heads, embedding_size, ffn_size, attention_dropout, relu_dropout))
def __init__(self, n_heads, n_layers, embedding_size, ffn_size, vocabulary_size, embedding=None, attention_dropout=0.0, relu_dropout=0.0, embeddings_scale=True, learn_positional_embeddings=False, padding_idx=None):
    super().__init__()
    self.embedding_size = embedding_size
    self.ffn_size = ffn_size
    self.n_layers = n_layers
    self.n_heads = n_heads
    self.dim = embedding_size
    self.embeddings_scale = embeddings_scale
    self.out_dim = embedding_size
    assert ((embedding_size % n_heads) == 0), 'Transformer embedding size must be a multiple of n_heads'
    n_positions = 1024
    self.embeddings = embedding
    self.position_embeddings = nn.Embedding(n_positions, embedding_size)
    if (not learn_positional_embeddings):
        create_position_codes(n_positions, embedding_size, out=self.position_embeddings.weight)
    else:
        nn.init.normal_(self.position_embeddings.weight, 0, (embedding_size ** (- 0.5)))
    self.layers = nn.ModuleList()
    for _ in range(self.n_layers):
        self.layers.append(TransformerDecoderLayer(n_heads, embedding_size, ffn_size, attention_dropout, relu_dropout))