




"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


lowerCAmelCase							:			Tuple     							=							{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase							:			Optional[Any]     							=							{
    """vocab_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-generator""": """https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt""",
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-generator""": (
            """https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCAmelCase							:			List[str]     							=							{
    """google/electra-small-generator""": 512,
    """google/electra-base-generator""": 512,
    """google/electra-large-generator""": 512,
    """google/electra-small-discriminator""": 512,
    """google/electra-base-discriminator""": 512,
    """google/electra-large-discriminator""": 512,
}

lowerCAmelCase							:			Optional[Any]     							=							{
    """google/electra-small-generator""": {"""do_lower_case""": True},
    """google/electra-base-generator""": {"""do_lower_case""": True},
    """google/electra-large-generator""": {"""do_lower_case""": True},
    """google/electra-small-discriminator""": {"""do_lower_case""": True},
    """google/electra-base-discriminator""": {"""do_lower_case""": True},
    """google/electra-large-discriminator""": {"""do_lower_case""": True},
}






class     __magic_name__							(       UpperCAmelCase__				):






				'''simple docstring'''
				__UpperCamelCase			  =						VOCAB_FILES_NAMES
				__UpperCamelCase			  =						PRETRAINED_VOCAB_FILES_MAP
				__UpperCamelCase			  =						PRETRAINED_INIT_CONFIGURATION
				__UpperCamelCase			  =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__UpperCamelCase			  =						ElectraTokenizer





				def __init__(       self		, _a=None		, _a=None		, _a=True		, _a="[UNK]"		, _a="[SEP]"		, _a="[PAD]"		, _a="[CLS]"		, _a="[MASK]"		, _a=True		, _a=None		, **_a		, ):





							"""simple docstring"""
							super().__init__(
							    _a		, tokenizer_file=_a		, do_lower_case=_a		, unk_token=_a		, sep_token=_a		, pad_token=_a		, cls_token=_a		, mask_token=_a		, tokenize_chinese_chars=_a		, strip_accents=_a		, **_a		, )

							lowerCamelCase			      =					json.loads(self.backend_tokenizer.normalizer.__getstate__() )
							if (
							    normalizer_state.get("""lowercase"""		, _a ) != do_lower_case
							    or normalizer_state.get("""strip_accents"""		, _a ) != strip_accents
							    or normalizer_state.get("""handle_chinese_chars"""		, _a ) != tokenize_chinese_chars
							):
										lowerCamelCase			      =					getattr(_a		, normalizer_state.pop("""type""" ) )
										lowerCamelCase			      =					do_lower_case
										lowerCamelCase			      =					strip_accents
										lowerCamelCase			      =					tokenize_chinese_chars
										lowerCamelCase			      =					normalizer_class(**_a )

							lowerCamelCase			      =					do_lower_case





				def    _lowerCAmelCase				(       self		, _a		, _a=None ):





							"""simple docstring"""
							lowerCamelCase			      =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
										output += token_ids_a + [self.sep_token_id]

							return output





				def    _lowerCAmelCase				(       self		, _a		, _a = None ):





							"""simple docstring"""
							lowerCamelCase			      =					[self.sep_token_id]
							lowerCamelCase			      =					[self.cls_token_id]
							if token_ids_a is None:
										return len(cls + token_ids_a + sep ) * [0]
							return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]





				def    _lowerCAmelCase				(       self		, _a		, _a = None ):





							"""simple docstring"""
							lowerCamelCase			      =					self._tokenizer.model.save(_a		, name=_a )
							return tuple(_a )









"""simple docstring"""




from math import ceil



def     a__ (				snake_case__       , snake_case__				)						->					Optional[int]:
			lowerCamelCase			      =					list(range(0       , snake_case__				)				)

			lowerCamelCase			      =					[item for sublist in list(device_map.values()				) for item in sublist]

			# Duplicate check
			lowerCamelCase			      =					[]
			for i in device_map_blocks:
						if device_map_blocks.count(snake_case__				) > 1 and i not in duplicate_blocks:
									duplicate_blocks.append(snake_case__				)
    # Missing blocks
			lowerCamelCase			      =					[i for i in blocks if i not in device_map_blocks]
			lowerCamelCase			      =					[i for i in device_map_blocks if i not in blocks]

			if len(snake_case__				) != 0:
						raise ValueError(
						    """Duplicate attention blocks specified in device_map. Attention blocks must be specified to one device."""
						    """ These attention blocks were specified more than once: """ + str(snake_case__				)				)
			if len(snake_case__				) != 0:
						raise ValueError(
						    """There are attention blocks for this model that are not specified in the device_map. Add these attention """
						    """blocks to a device on the device_map: """ + str(snake_case__				)				)
			if len(snake_case__				) != 0:
						raise ValueError(
						    """The device_map contains more attention blocks than this model has. Remove these from the device_map:"""
						    + str(snake_case__				)				)



def     a__ (				snake_case__       , snake_case__				)						->					List[Any]:
			lowerCamelCase			      =					list(range(snake_case__				)				)
			lowerCamelCase			      =					int(ceil(n_layers / len(snake_case__				)				)				)
			lowerCamelCase			      =					[layers[i : i + n_blocks] for i in range(0       , snake_case__       , snake_case__				)]

			return dict(zip(snake_case__       , snake_case__				)				)



