def __init__(self, vocab_size, hidden_size, num_hidden_layers, num_attention_heads, intermediate_size, hidden_act, dropout_rate=None, embedding_size=None, attention_head_size=None, attention_key_size=None, sequence_length=None, keep_tokens=None, compound_tokens=None, residual_attention_scores=False, layers=None, prefix=None, name=None, **kwargs):
    if (keep_tokens is not None):
        vocab_size = len(keep_tokens)
    if (compound_tokens is not None):
        vocab_size += len(compound_tokens)
    self.vocab_size = vocab_size
    self.hidden_size = hidden_size
    self.num_hidden_layers = num_hidden_layers
    self.num_attention_heads = num_attention_heads
    self.attention_head_size = (attention_head_size or (hidden_size // num_attention_heads))
    self.attention_key_size = (attention_key_size or self.attention_head_size)
    self.intermediate_size = intermediate_size
    self.dropout_rate = (dropout_rate or 0)
    self.hidden_act = hidden_act
    self.embedding_size = (embedding_size or hidden_size)
    self.sequence_length = sequence_length
    self.keep_tokens = keep_tokens
    self.compound_tokens = compound_tokens
    self.attention_bias = None
    self.position_bias = None
    self.attention_scores = None
    self.residual_attention_scores = residual_attention_scores
    self.layers = ({} if (layers is None) else layers)
    self.prefix = (prefix or '')
    self.name = name
    self.built = False
def __init__(self, vocab_size, hidden_size, num_hidden_layers, num_attention_heads, intermediate_size, hidden_act, dropout_rate=None, embedding_size=None, attention_head_size=None, attention_key_size=None, sequence_length=None, keep_tokens=None, compound_tokens=None, residual_attention_scores=False, layers=None, prefix=None, name=None, **kwargs):
    if (keep_tokens is not None):
        vocab_size = len(keep_tokens)
    if (compound_tokens is not None):
        vocab_size += len(compound_tokens)
    self.vocab_size = vocab_size
    self.hidden_size = hidden_size
    self.num_hidden_layers = num_hidden_layers
    self.num_attention_heads = num_attention_heads
    self.attention_head_size = (attention_head_size or (hidden_size // num_attention_heads))
    self.attention_key_size = (attention_key_size or self.attention_head_size)
    self.intermediate_size = intermediate_size
    self.dropout_rate = (dropout_rate or 0)
    self.hidden_act = hidden_act
    self.embedding_size = (embedding_size or hidden_size)
    self.sequence_length = sequence_length
    self.keep_tokens = keep_tokens
    self.compound_tokens = compound_tokens
    self.attention_bias = None
    self.position_bias = None
    self.attention_scores = None
    self.residual_attention_scores = residual_attention_scores
    self.layers = ({} if (layers is None) else layers)
    self.prefix = (prefix or )
    self.name = name
    self.built = False