def __init__(self, attention_type, config):
    super().__init__()
    self.window_size = None
    max_positions = config.max_position_embeddings
    bias = torch.tril(torch.ones((max_positions, max_positions), dtype=torch.uint8)).view(1, 1, max_positions, max_positions).bool()
    if (attention_type == 'local'):
        self.register_buffer('bias', (bias ^ torch.tril(bias, (- config.window_size))))
    else:
        self.register_buffer('bias', bias)
    self.register_buffer('masked_bias', torch.tensor((- 1000000000.0)))
    self.attn_dropout = nn.Dropout(config.attention_dropout)
    self.resid_dropout = nn.Dropout(config.resid_dropout)
    self.embed_dim = config.hidden_size
    self.num_heads = config.num_heads
    self.head_dim = (self.embed_dim // self.num_heads)
    if ((self.head_dim * self.num_heads) != self.embed_dim):
        raise ValueError(f'embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`: {self.num_heads}).')
    self.k_proj = LazyLinearAPICompatible(self.embed_dim, self.embed_dim, bias=False)
    self.v_proj = LazyLinearAPICompatible(self.embed_dim, self.embed_dim, bias=False)
    self.q_proj = LazyLinearAPICompatible(self.embed_dim, self.embed_dim, bias=False)
    self.out_proj = LazyLinearAPICompatible(self.embed_dim, self.embed_dim, bias=True)
def __init__(self, attention_type, config):
    super().__init__()
    self.window_size = None
    max_positions = config.max_position_embeddings
    bias = torch.tril(torch.ones((max_positions, max_positions), dtype=torch.uint8)).view(1, 1, max_positions, max_positions).bool()
    if (attention_type == 'local'):
        self.register_buffer('bias', (bias ^ torch.tril(bias, (- config.window_size))))
    else:
        self.register_buffer('bias', bias)
    self.register_buffer('masked_bias', torch.tensor((- 1000000000.0)))
    self.attn_dropout = nn.Dropout(config.attention_dropout)
    self.resid_dropout = nn.Dropout(config.resid_dropout)
    self.embed_dim = config.hidden_size
    self.num_heads = config.num_heads
    self.head_dim = (self.embed_dim // self.num_heads)
    if ((self.head_dim * self.num_heads) != self.embed_dim):
        raise ValueError(f'embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`: {self.num_heads}).')
    self.k_proj = LazyLinearAPICompatible(self.embed_dim, self.embed_dim, bias=False)
    self.v_proj = LazyLinearAPICompatible(self.embed_dim, self.embed_dim, bias=False)
    self.q_proj = LazyLinearAPICompatible(self.embed_dim, self.embed_dim, bias=False)
    self.out_proj = LazyLinearAPICompatible(self.embed_dim, self.embed_dim, bias=True)