def __init__(self, enc_attn_dim, vocab_size, use_checkpoint, clip=False, dropout=0.0):
    super().__init__()
    d_model = enc_attn_dim
    self.use_checkpoint = use_checkpoint
    self.encoder = nn.ModuleList([])
    for _ in range((d_model // 64)):
        self.encoder += [AttentionBlock(d_model, num_heads=(d_model // 64), norm_type='layernorm', cross=False, use_checkpoint=use_checkpoint, dropout=dropout)]
        self.encoder += [FFN(d_model, dropout=dropout)]
    self.pos_enc = PositionalEncoding(d_model, dropout=dropout)
    self.emb = nn.Embedding(vocab_size, d_model)
    if clip:
        self.clip_proj = conv_nd(1, enc_attn_dim, enc_attn_dim, 1)
def __init__(self, enc_attn_dim, vocab_size, use_checkpoint, clip=False, dropout=0.0):
    super().__init__()
    d_model = enc_attn_dim
    self.use_checkpoint = use_checkpoint
    self.encoder = nn.ModuleList([])
    for _ in range((d_model // 64)):
        self.encoder += [AttentionBlock(d_model, num_heads=(d_model // 64), norm_type='layernorm', cross=False, use_checkpoint=use_checkpoint, dropout=dropout)]
        self.encoder += [FFN(d_model, dropout=dropout)]
    self.pos_enc = PositionalEncoding(d_model, dropout=dropout)
    self.emb = nn.Embedding(vocab_size, d_model)
    if clip:
        self.clip_proj = conv_nd(1, enc_attn_dim, enc_attn_dim, 1)