def forward(self, input_states, attention_mask, segments=None, head_mask=None, output_all_states=False, output_attentions=False, keep_multihead_output=False):
    if (attention_mask is None):
        attention_mask = torch.ones(input_states.size(0), input_states.size(1))
    if (len(attention_mask.size()) == 3):
        extended_attention_mask = attention_mask.unsqueeze(1)
    elif (len(attention_mask.size()) == 2):
        extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
    extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype, device=input_states.device)
    extended_attention_mask = ((1.0 - extended_attention_mask) * (- 10000.0))
    attention_mask = extended_attention_mask
    if (head_mask is not None):
        if (head_mask.dim() == 1):
            head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze((- 1)).unsqueeze((- 1))
            head_mask = head_mask.expand_as(self.num_hidden_layers, (- 1), (- 1), (- 1), (- 1))
        elif (head_mask.dim() == 2):
            head_mask = head_mask.unsqueeze(1).unsqueeze((- 1)).unsqueeze((- 1))
        head_mask = head_mask.to(dtype=next(self.parameters()).dtype, device=input_states.device)
    else:
        head_mask = None
    input_states = self.embedding(input_states)
    if (self.pos_embedding is not None):
        pos_states = self.pos_embedding(self.get_pos_states(input_states))
        input_states = (input_states + pos_states.to(device=input_states.device))
    if ((self.segment_embedding is not None) and (segments is not None)):
        segment_states = self.segment_embedding(segments)
        input_states = (input_states + segment_states)
    input_states = self.embed_refine_net(input_states)
    output_states = self.encoder(input_states, attention_mask, head_mask, output_all_states, output_attentions, keep_multihead_output)
    if output_attentions:
        (output_states, attention_probs) = output_states
        return (output_states[(- 1)], attention_probs)
    return output_states[(- 1)]
def forward(self, input_states, attention_mask, segments=None, head_mask=None, output_all_states=False, output_attentions=False, keep_multihead_output=False):
    if (attention_mask is None):
        attention_mask = torch.ones(input_states.size(0), input_states.size(1))
    if (len(attention_mask.size()) == 3):
        extended_attention_mask = attention_mask.unsqueeze(1)
    elif (len(attention_mask.size()) == 2):
        extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
    extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype, device=input_states.device)
    extended_attention_mask = ((1.0 - extended_attention_mask) * (- 10000.0))
    attention_mask = extended_attention_mask
    if (head_mask is not None):
        if (head_mask.dim() == 1):
            head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze((- 1)).unsqueeze((- 1))
            head_mask = head_mask.expand_as(self.num_hidden_layers, (- 1), (- 1), (- 1), (- 1))
        elif (head_mask.dim() == 2):
            head_mask = head_mask.unsqueeze(1).unsqueeze((- 1)).unsqueeze((- 1))
        head_mask = head_mask.to(dtype=next(self.parameters()).dtype, device=input_states.device)
    else:
        head_mask = None
    input_states = self.embedding(input_states)
    if (self.pos_embedding is not None):
        pos_states = self.pos_embedding(self.get_pos_states(input_states))
        input_states = (input_states + pos_states.to(device=input_states.device))
    if ((self.segment_embedding is not None) and (segments is not None)):
        segment_states = self.segment_embedding(segments)
        input_states = (input_states + segment_states)
    input_states = self.embed_refine_net(input_states)
    output_states = self.encoder(input_states, attention_mask, head_mask, output_all_states, output_attentions, keep_multihead_output)
    if output_attentions:
        (output_states, attention_probs) = output_states
        return (output_states[(- 1)], attention_probs)
    return output_states[(- 1)]