





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


a__:      Tuple									=		{
    '''configuration_mega''': ['''MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''MegaConfig''', '''MegaOnnxConfig'''],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a__:      List[Any]									=		[
           '''MEGA_PRETRAINED_MODEL_ARCHIVE_LIST''',
           '''MegaForCausalLM''',
           '''MegaForMaskedLM''',
           '''MegaForMultipleChoice''',
           '''MegaForQuestionAnswering''',
           '''MegaForSequenceClassification''',
           '''MegaForTokenClassification''',
           '''MegaModel''',
           '''MegaPreTrainedModel''',
       ]

if TYPE_CHECKING:
       from .configuration_mega import MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP, MegaConfig, MegaOnnxConfig

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_mega import (
                  MEGA_PRETRAINED_MODEL_ARCHIVE_LIST,
                  MegaForCausalLM,
                  MegaForMaskedLM,
                  MegaForMultipleChoice,
                  MegaForQuestionAnswering,
                  MegaForSequenceClassification,
                  MegaForTokenClassification,
                  MegaModel,
                  MegaPreTrainedModel,
              )

else:
       import sys

       a__:      Optional[Any]									=		_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)












"""simple docstring"""




from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


a						: List[str]     	=			logging.get_logger(__name__)

a						: List[Any]     	=			{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

a						: str     	=			{
    '''vocab_file''': {
        '''allegro/herbert-base-cased''': '''https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json'''
    },
    '''merges_file''': {
        '''allegro/herbert-base-cased''': '''https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt'''
    },
}

a						: Tuple     	=			{'''allegro/herbert-base-cased''': 514}
a						: Optional[int]     	=			{}


class        __UpperCamelCase				(						a__						):
							lowerCamelCase   :					str      					=VOCAB_FILES_NAMES
							lowerCamelCase   :					Dict      					=PRETRAINED_VOCAB_FILES_MAP
							lowerCamelCase   :					Dict      					=PRETRAINED_INIT_CONFIGURATION
							lowerCamelCase   :					int      					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCamelCase   :					Optional[Any]      					=HerbertTokenizer






							def __init__(   self      ,					lowerCAmelCase__=None      ,					lowerCAmelCase__=None      ,					lowerCAmelCase__=None      ,					lowerCAmelCase__="<s>"      ,					lowerCAmelCase__="<unk>"      ,					lowerCAmelCase__="<pad>"      ,					lowerCAmelCase__="<mask>"      ,					lowerCAmelCase__="</s>"      ,					**lowerCAmelCase__      ,					)	-> Optional[int]:
								super().__init__(
								    lowerCAmelCase__      ,					lowerCAmelCase__      ,					tokenizer_file=lowerCAmelCase__      ,					cls_token=lowerCAmelCase__      ,					unk_token=lowerCAmelCase__      ,					pad_token=lowerCAmelCase__      ,					mask_token=lowerCAmelCase__      ,					sep_token=lowerCAmelCase__      ,					**lowerCAmelCase__      ,					)






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__ = None      )	-> List[int]:
								a      :		Optional[Any]	=				[self.cls_token_id]
								a      :		Any	=				[self.sep_token_id]
								if token_ids_a is None:
									return cls + token_ids_a + sep

								return cls + token_ids_a + sep + token_ids_a + sep






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__ = None      ,					lowerCAmelCase__ = False      )	-> List[int]:
								if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=lowerCAmelCase__      ,					token_ids_a=lowerCAmelCase__      ,					already_has_special_tokens=lowerCAmelCase__      )

								if token_ids_a is None:
									return [1] + ([0] * len(lowerCAmelCase__      )) + [1]
								return [1] + ([0] * len(lowerCAmelCase__      )) + [1] + ([0] * len(lowerCAmelCase__      )) + [1]






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__ = None      )	-> List[int]:
								a      :		Dict	=				[self.sep_token_id]
								a      :		Tuple	=				[self.cls_token_id]

								if token_ids_a is None:
									return len(cls + token_ids_a + sep      ) * [0]
								return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__ = None      )	-> Tuple[str]:
								a      :		List[str]	=				self._tokenizer.model.save(lowerCAmelCase__      ,					name=lowerCAmelCase__      )
								return tuple(lowerCAmelCase__      )


