



"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ....tokenization_utils_fast import PreTrainedTokenizerFast
from ....utils import logging
from .tokenization_retribert import RetriBertTokenizer


_UpperCamelCase: List[str]										=     logging.get_logger(__name__)

_UpperCamelCase: str										=     {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_UpperCamelCase: str										=     {
    """vocab_file""": {
        """yjernite/retribert-base-uncased""": (
            """https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """yjernite/retribert-base-uncased""": (
            """https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

_UpperCamelCase: str										=     {
    """yjernite/retribert-base-uncased""": 5_1_2,
}


_UpperCamelCase: List[str]										=     {
    """yjernite/retribert-base-uncased""": {"""do_lower_case""": True},
}


class      a__						(  UpperCAmelCase__   ):
   _lowerCamelCase    							= VOCAB_FILES_NAMES
   _lowerCamelCase    							= PRETRAINED_VOCAB_FILES_MAP
   _lowerCamelCase    							= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _lowerCamelCase    							= PRETRAINED_INIT_CONFIGURATION
   _lowerCamelCase    							= RetriBertTokenizer
   _lowerCamelCase    							= ["""input_ids""", """attention_mask"""]


   def __init__(							self					:							Tuple,   lowerCAmelCase					:							str=None,   lowerCAmelCase					:							Optional[int]=None,   lowerCAmelCase					:							int=True,   lowerCAmelCase					:							Union[str, Any]="[UNK]",   lowerCAmelCase					:							str="[SEP]",   lowerCAmelCase					:							int="[PAD]",   lowerCAmelCase					:							Union[str, Any]="[CLS]",   lowerCAmelCase					:							List[Any]="[MASK]",   lowerCAmelCase					:							Any=True,   lowerCAmelCase					:							Optional[Any]=None,   **lowerCAmelCase					:							Dict,   )			->				Union[str, Any]:
         super().__init__(
             __lowercase,   tokenizer_file=__lowercase,   do_lower_case=__lowercase,   unk_token=__lowercase,   sep_token=__lowercase,   pad_token=__lowercase,   cls_token=__lowercase,   mask_token=__lowercase,   tokenize_chinese_chars=__lowercase,   strip_accents=__lowercase,   **__lowercase,   )

         lowercase     :					Dict 		=      json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
         if (
             normalizer_state.get('lowercase',   __lowercase	) != do_lower_case
             or normalizer_state.get('strip_accents',   __lowercase	) != strip_accents
             or normalizer_state.get('handle_chinese_chars',   __lowercase	) != tokenize_chinese_chars
         ):
               lowercase     :					List[str] 		=      getattr(__lowercase,   normalizer_state.pop('type'	)	)
               lowercase     :					str 		=      do_lower_case
               lowercase     :					Tuple 		=      strip_accents
               lowercase     :					Tuple 		=      tokenize_chinese_chars
               lowercase     :					Tuple 		=      normalizer_class(**__lowercase	)

         lowercase     :					int 		=      do_lower_case


   def 				lowercase (							self					:							List[Any],   lowerCAmelCase					:							Dict,   lowerCAmelCase					:							Union[str, Any]=None	)			->				Optional[int]:
         lowercase     :					Union[str, Any] 		=      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

         return output


   def 				lowercase (							self					:							Optional[int],   lowerCAmelCase					:							Dict,   lowerCAmelCase					:							Union[str, Any] = None	)			->				List[int]:
         lowercase     :					List[Any] 		=      [self.sep_token_id]
         lowercase     :					Union[str, Any] 		=      [self.cls_token_id]
         if token_ids_a is None:
               return len(cls + token_ids_a + sep	) * [0]
         return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]






   def 				lowercase (							self					:							str,   lowerCAmelCase					:							str,   lowerCAmelCase					:							Any = None	)			->				Tuple[str]:
         lowercase     :					Any 		=      self._tokenizer.model.save(__lowercase,   name=__lowercase	)
         return tuple(__lowercase	)




from .glue import glue_convert_examples_to_features, glue_output_modes, glue_processors, glue_tasks_num_labels
from .squad import SquadExample, SquadFeatures, SquadVaProcessor, SquadVaProcessor, squad_convert_examples_to_features
from .utils import DataProcessor, InputExample, InputFeatures, SingleSentenceClassificationProcessor
from .xnli import xnli_output_modes, xnli_processors, xnli_tasks_num_labels