def __init__(self, config, **kwargs):
    super().__init__(**kwargs)
    if ((config.hidden_size % config.num_attention_heads) != 0):
        raise ValueError(f'The hidden size ({config.hidden_size}) is not a multiple of the number of attention heads ({config.num_attention_heads})')
    new_num_attention_heads = int((config.num_attention_heads / config.head_ratio))
    if (new_num_attention_heads < 1):
        self.head_ratio = config.num_attention_heads
        num_attention_heads = 1
    else:
        num_attention_heads = new_num_attention_heads
        self.head_ratio = config.head_ratio
    self.num_attention_heads = num_attention_heads
    self.conv_kernel_size = config.conv_kernel_size
    assert ((config.hidden_size % self.num_attention_heads) == 0), 'hidden_size should be divisible by num_attention_heads'
    self.attention_head_size = (config.hidden_size // config.num_attention_heads)
    self.all_head_size = (self.num_attention_heads * self.attention_head_size)
    self.query = tf.keras.layers.Dense(self.all_head_size, kernel_initializer=get_initializer(config.initializer_range), name='query')
    self.key = tf.keras.layers.Dense(self.all_head_size, kernel_initializer=get_initializer(config.initializer_range), name='key')
    self.value = tf.keras.layers.Dense(self.all_head_size, kernel_initializer=get_initializer(config.initializer_range), name='value')
    self.key_conv_attn_layer = tf.keras.layers.SeparableConv1D(self.all_head_size, self.conv_kernel_size, padding='same', activation=None, depthwise_initializer=get_initializer((1 / self.conv_kernel_size)), pointwise_initializer=get_initializer(config.initializer_range), name='key_conv_attn_layer')
    self.conv_kernel_layer = tf.keras.layers.Dense((self.num_attention_heads * self.conv_kernel_size), activation=None, name='conv_kernel_layer', kernel_initializer=get_initializer(config.initializer_range))
    self.conv_out_layer = tf.keras.layers.Dense(self.all_head_size, activation=None, name='conv_out_layer', kernel_initializer=get_initializer(config.initializer_range))
    self.dropout = tf.keras.layers.Dropout(config.attention_probs_dropout_prob)
def __init__(self, config, **kwargs):
    super().__init__(**kwargs)
    if ((config.hidden_size % config.num_attention_heads) != 0):
        raise ValueError(f'The hidden size ({config.hidden_size}) is not a multiple of the number of attention heads ({config.num_attention_heads})')
    new_num_attention_heads = int((config.num_attention_heads / config.head_ratio))
    if (new_num_attention_heads < 1):
        self.head_ratio = config.num_attention_heads
        num_attention_heads = 1
    else:
        num_attention_heads = new_num_attention_heads
        self.head_ratio = config.head_ratio
    self.num_attention_heads = num_attention_heads
    self.conv_kernel_size = config.conv_kernel_size
    assert ((config.hidden_size % self.num_attention_heads) == 0), 'hidden_size should be divisible by num_attention_heads'
    self.attention_head_size = (config.hidden_size // config.num_attention_heads)
    self.all_head_size = (self.num_attention_heads * self.attention_head_size)
    self.query = tf.keras.layers.Dense(self.all_head_size, kernel_initializer=get_initializer(config.initializer_range), name='query')
    self.key = tf.keras.layers.Dense(self.all_head_size, kernel_initializer=get_initializer(config.initializer_range), name='key')
    self.value = tf.keras.layers.Dense(self.all_head_size, kernel_initializer=get_initializer(config.initializer_range), name='value')
    self.key_conv_attn_layer = tf.keras.layers.SeparableConv1D(self.all_head_size, self.conv_kernel_size, padding='same', activation=None, depthwise_initializer=get_initializer((1 / self.conv_kernel_size)), pointwise_initializer=get_initializer(config.initializer_range), name='key_conv_attn_layer')
    self.conv_kernel_layer = tf.keras.layers.Dense((self.num_attention_heads * self.conv_kernel_size), activation=None, name='conv_kernel_layer', kernel_initializer=get_initializer(config.initializer_range))
    self.conv_out_layer = tf.keras.layers.Dense(self.all_head_size, activation=None, name='conv_out_layer', kernel_initializer=get_initializer(config.initializer_range))
    self.dropout = tf.keras.layers.Dropout(config.attention_probs_dropout_prob)