





"""simple docstring"""




import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import TaBlock, TaConfig, TaLayerNorm

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin

class  _UpperCAmelCase  (   UpperCAmelCase__			,   UpperCAmelCase__			,   UpperCAmelCase__		):



      '''simple docstring'''

      @register_to_config
      def __init__(     self					,      snake_case_					,      snake_case_					,      snake_case_					,      snake_case_					,      snake_case_					,      snake_case_					,      snake_case_					,      snake_case_					,      snake_case_					,      snake_case_ = False					,      ):

        """simple docstring"""

        super().__init__()

        A_  :       str										=  nn.Embedding(snake_case_					,      snake_case_  )

        A_  :       Dict										=  nn.Embedding(snake_case_					,      snake_case_  )
        A_  :       Optional[int]										=  False

        A_  :       Optional[Any]										=  nn.Dropout(p=snake_case_  )

        A_  :       Union[str, Any]										=  TaConfig(
            vocab_size=snake_case_					,      d_model=snake_case_					,      num_heads=snake_case_					,      d_kv=snake_case_					,      d_ff=snake_case_					,      dropout_rate=snake_case_					,      feed_forward_proj=snake_case_					,      is_decoder=snake_case_					,      is_encoder_decoder=snake_case_					,      )

        A_  :       Any										=  nn.ModuleList()
        for lyr_num in range(snake_case_  ):
          A_  :       Optional[int]										=  TaBlock(snake_case_  )
          self.encoders.append(snake_case_  )

        A_  :       Any										=  TaLayerNorm(snake_case_  )
        A_  :       List[Any]										=  nn.Dropout(p=snake_case_  )

      def 					lowerCamelCase_			(     self					,      snake_case_					,      snake_case_  ):

        """simple docstring"""

        A_  :       List[str]										=  self.token_embedder(snake_case_  )

        A_  :       Any										=  encoder_input_tokens.shape[1]
        A_  :       Dict										=  torch.arange(snake_case_					,      device=encoder_input_tokens.device  )
        x += self.position_encoding(snake_case_  )

        A_  :       str										=  self.dropout_pre(snake_case_  )

        # inverted the attention mask
        A_  :       List[Any]										=  encoder_input_tokens.size()
        A_  :       Optional[Any]										=  self.get_extended_attention_mask(snake_case_					,      snake_case_  )

        for lyr in self.encoders:
          A_  :       List[Any]										=  lyr(snake_case_					,      snake_case_  )[0]
        A_  :       Optional[Any]										=  self.layer_norm(snake_case_  )

        return self.dropout_post(snake_case_  ), encoder_inputs_mask






"""simple docstring"""




import os





def 						UpperCAmelCase__		(     ):






  """simple docstring"""





  A_  :       Any										=  os.path.join(os.path.dirname(_UpperCAmelCase					) ,					'num.txt'					)
  with open(_UpperCAmelCase					) as file_hand:
    return str(sum(int(_UpperCAmelCase					) for line in file_hand					)					)[:10]


if __name__ == "__main__":
   print(solution())