




'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_squeezebert import SqueezeBertTokenizer


_SCREAMING_SNAKE_CASE							:   Tuple      		= logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE							:   Tuple      		= {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_SCREAMING_SNAKE_CASE							:   str      		= {
    "vocab_file": {
        "squeezebert/squeezebert-uncased": (
            "https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/vocab.txt"
        ),
        "squeezebert/squeezebert-mnli": "https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/vocab.txt",
        "squeezebert/squeezebert-mnli-headless": (
            "https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "squeezebert/squeezebert-uncased": (
            "https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/tokenizer.json"
        ),
        "squeezebert/squeezebert-mnli": (
            "https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/tokenizer.json"
        ),
        "squeezebert/squeezebert-mnli-headless": (
            "https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/tokenizer.json"
        ),
    },
}

_SCREAMING_SNAKE_CASE							:   List[Any]      		= {
    "squeezebert/squeezebert-uncased": 512,
    "squeezebert/squeezebert-mnli": 512,
    "squeezebert/squeezebert-mnli-headless": 512,
}


_SCREAMING_SNAKE_CASE							:   str      		= {
    "squeezebert/squeezebert-uncased": {"do_lower_case": True},
    "squeezebert/squeezebert-mnli": {"do_lower_case": True},
    "squeezebert/squeezebert-mnli-headless": {"do_lower_case": True},
}




class  _snake_case		(	lowercase_							):
			lowerCAmelCase_     :      Tuple						       =    VOCAB_FILES_NAMES
			lowerCAmelCase_     :      Union[str, Any]						       =    PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase_     :      Union[str, Any]						       =    PRETRAINED_INIT_CONFIGURATION
			lowerCAmelCase_     :      str						       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase_     :      Dict						       =    SqueezeBertTokenizer




			def __init__(					self				,							a__=None				,							a__=None				,							a__=True				,							a__="[UNK]"				,							a__="[SEP]"				,							a__="[PAD]"				,							a__="[CLS]"				,							a__="[MASK]"				,							a__=True				,							a__=None				,							**a__				,							)				->				Dict:

										'''simple docstring'''




										super().__init__(
										    a__				,							tokenizer_file=a__				,							do_lower_case=a__				,							unk_token=a__				,							sep_token=a__				,							pad_token=a__				,							cls_token=a__				,							mask_token=a__				,							tokenize_chinese_chars=a__				,							strip_accents=a__				,							**a__				,							)

										snake_case_						=      json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
										if (
										    normalizer_state.get("lowercase"				,							a__							) != do_lower_case
										    or normalizer_state.get("strip_accents"				,							a__							) != strip_accents
										    or normalizer_state.get("handle_chinese_chars"				,							a__							) != tokenize_chinese_chars
										):
																	snake_case_						=      getattr(a__				,							normalizer_state.pop("type"							)							)
																	snake_case_						=      do_lower_case
																	snake_case_						=      strip_accents
																	snake_case_						=      tokenize_chinese_chars
																	snake_case_						=      normalizer_class(**a__							)

										snake_case_						=      do_lower_case




			def    lowerCAmelCase__				(					self				,							a__				,							a__=None							)				->				Union[str, Any]:

										'''simple docstring'''




										snake_case_						=      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
																	output += token_ids_a + [self.sep_token_id]

										return output




			def    lowerCAmelCase__				(					self				,							a__				,							a__ = None							)				->				List[int]:

										'''simple docstring'''




										snake_case_						=      [self.sep_token_id]
										snake_case_						=      [self.cls_token_id]
										if token_ids_a is None:
																	return len(cls + token_ids_a + sep							) * [0]
										return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]




			def    lowerCAmelCase__				(					self				,							a__				,							a__ = None							)				->				Tuple[str]:

										'''simple docstring'''




										snake_case_						=      self._tokenizer.model.save(a__				,							name=a__							)
										return tuple(a__							)






'''simple docstring'''







import json
import os
import tempfile

from transformers.testing_utils import check_json_file_has_correct_format






class  lowerCAmelCase__   :
					lowerCAmelCase_        =     None






					def 	_snake_case    (   self		):


											"""simple docstring"""





											lowercase_	:     Tuple													=       self.feature_extraction_class(**self.feat_extract_dict		)
											lowercase_	:     Any													=       json.loads(feat_extract.to_json_string()		)
											for key, value in self.feat_extract_dict.items():
																	self.assertEqual(obj[key]				,							__SCREAMING_SNAKE_CASE		)






					def 	_snake_case    (   self		):


											"""simple docstring"""





											lowercase_	:     str													=       self.feature_extraction_class(**self.feat_extract_dict		)

											with tempfile.TemporaryDirectory() as tmpdirname:
																	lowercase_	:     str													=       os.path.join(__SCREAMING_SNAKE_CASE				,							'''feat_extract.json'''		)
																	feat_extract_first.to_json_file(__SCREAMING_SNAKE_CASE		)
																	lowercase_	:     str													=       self.feature_extraction_class.from_json_file(__SCREAMING_SNAKE_CASE		)

											self.assertEqual(feat_extract_second.to_dict()				,							feat_extract_first.to_dict()		)






					def 	_snake_case    (   self		):


											"""simple docstring"""





											lowercase_	:     Tuple													=       self.feature_extraction_class(**self.feat_extract_dict		)

											with tempfile.TemporaryDirectory() as tmpdirname:
																	lowercase_	:     Union[str, Any]													=       feat_extract_first.save_pretrained(__SCREAMING_SNAKE_CASE		)[0]
																	check_json_file_has_correct_format(__SCREAMING_SNAKE_CASE		)
																	lowercase_	:     str													=       self.feature_extraction_class.from_pretrained(__SCREAMING_SNAKE_CASE		)

											self.assertEqual(feat_extract_second.to_dict()				,							feat_extract_first.to_dict()		)






					def 	_snake_case    (   self		):


											"""simple docstring"""





											lowercase_	:     Optional[Any]													=       self.feature_extraction_class()
											self.assertIsNotNone(__SCREAMING_SNAKE_CASE		)



