'''simple docstring'''

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


UpperCamelCase__  :						Dict					=      logging.get_logger(__name__)

UpperCamelCase__  :						List[Any]					=      {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCamelCase__  :						Optional[Any]					=      {
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

UpperCamelCase__  :						List[Any]					=      {
    'junnyu/roformer_chinese_small': 1_536,
    'junnyu/roformer_chinese_base': 1_536,
    'junnyu/roformer_chinese_char_small': 512,
    'junnyu/roformer_chinese_char_base': 512,
    'junnyu/roformer_small_discriminator': 128,
    'junnyu/roformer_small_generator': 128,
}


UpperCamelCase__  :						Optional[int]					=      {
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}


class 					_lowercase						(   lowerCAmelCase ):
      '''simple docstring'''


      UpperCAmelCase_   :   List[str]   				= VOCAB_FILES_NAMES
      UpperCAmelCase_   :   Any   				= PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase_   :   Optional[int]   				= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCAmelCase_   :   Dict   				= PRETRAINED_INIT_CONFIGURATION
      UpperCAmelCase_   :   Dict   				= RoFormerTokenizer




      def __init__(					self       ,lowerCamelCase_=None       ,lowerCamelCase_=None       ,lowerCamelCase_=True       ,lowerCamelCase_="[UNK]"       ,lowerCamelCase_="[SEP]"       ,lowerCamelCase_="[PAD]"       ,lowerCamelCase_="[CLS]"       ,lowerCamelCase_="[MASK]"       ,lowerCamelCase_=True       ,lowerCamelCase_=None       ,**lowerCamelCase_       ,)      ->      Any:



         '''simple docstring'''



         super().__init__(
             lowerCamelCase_       ,tokenizer_file=lowerCamelCase_       ,do_lower_case=lowerCamelCase_       ,unk_token=lowerCamelCase_       ,sep_token=lowerCamelCase_       ,pad_token=lowerCamelCase_       ,cls_token=lowerCamelCase_       ,mask_token=lowerCamelCase_       ,tokenize_chinese_chars=lowerCamelCase_       ,strip_accents=lowerCamelCase_       ,**lowerCamelCase_       ,)

         UpperCAmelCase__				:					Any 					=	json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
         if (
             pre_tok_state.get('''lowercase'''       ,lowerCamelCase_				) != do_lower_case
             or pre_tok_state.get('''strip_accents'''       ,lowerCamelCase_				) != strip_accents
         ):
            UpperCAmelCase__				:					Any 					=	getattr(lowerCamelCase_       ,pre_tok_state.pop('''type'''				)				)
            UpperCAmelCase__				:					int 					=	do_lower_case
            UpperCAmelCase__				:					Dict 					=	strip_accents
            UpperCAmelCase__				:					Dict 					=	pre_tok_class(**lowerCamelCase_				)

         UpperCAmelCase__				:					Union[str, Any] 					=	do_lower_case




      def __getstate__(					self				)      ->      Optional[int]:



         '''simple docstring'''



         UpperCAmelCase__				:					Tuple 					=	self.__dict__.copy()
         UpperCAmelCase__				:					Union[str, Any] 					=	BertPreTokenizer()
         return state




      def __setstate__(					self       ,lowerCamelCase_				)      ->      str:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[int] 					=	d
         UpperCAmelCase__				:					Any 					=	self.__dict__['''_tokenizer'''].get_vocab()
         UpperCAmelCase__				:					List[str] 					=	PreTokenizer.custom(JiebaPreTokenizer(lowerCamelCase_				)				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_=None				)      ->      Optional[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					List[str] 					=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
            output += token_ids_a + [self.sep_token_id]

         return output




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_ = None				)      ->      List[int]:



         '''simple docstring'''



         UpperCAmelCase__				:					List[Any] 					=	[self.sep_token_id]
         UpperCAmelCase__				:					Optional[Any] 					=	[self.cls_token_id]
         if token_ids_a is None:
            return len(cls + token_ids_a + sep				) * [0]
         return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_ = None				)      ->      Tuple[str]:



         '''simple docstring'''



         UpperCAmelCase__				:					List[Any] 					=	self._tokenizer.model.save(lowerCamelCase_       ,name=lowerCamelCase_				)
         return tuple(lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_=None       ,lowerCamelCase_=None       ,lowerCamelCase_=False       ,**lowerCamelCase_       ,)      ->      Optional[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					List[Any] 					=	BertPreTokenizer()
         return super().save_pretrained(lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,**lowerCamelCase_				)





'''simple docstring'''

from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Features, Sequence, Value
from .base import TaskTemplate


@dataclass(frozen=lowerCAmelCase )
class 					_lowercase						(   lowerCAmelCase ):
      '''simple docstring'''


      UpperCAmelCase_   :   str   				= field(default='''question-answering-extractive'''		,metadata={'''include_in_asdict_even_if_is_default''': True} )
      UpperCAmelCase_   :   ClassVar[Features]   				= Features({'''question''': Value('''string''' ), '''context''': Value('''string''' )} )
      UpperCAmelCase_   :   ClassVar[Features]   				= Features(
          {
              '''answers''': Sequence(
                  {
                      '''text''': Value('''string''' ),
                      '''answer_start''': Value('''int32''' ),
                  } )
          } )
      UpperCAmelCase_   :   str   				= "question"
      UpperCAmelCase_   :   str   				= "context"
      UpperCAmelCase_   :   str   				= "answers"




      @property
      def 		lowerCAmelCase__       (					self				)      ->      Dict[str, str]:



         '''simple docstring'''



         return {self.question_column: "question", self.context_column: "context", self.answers_column: "answers"}




