"""simple docstring"""





import warnings
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__A     = logging.get_logger(__name__)
class        _snake_case   (    __UpperCAmelCase						):
						snake_case__	     =	["input_ids", "attention_mask"]






						def __init__(				self					:	Optional[int]							,    UpperCAmelCase					:	int="</s>"							,    UpperCAmelCase					:	str="<unk>"							,    UpperCAmelCase					:	Union[str, Any]="<pad>"							,    UpperCAmelCase					:	int=125							,    UpperCAmelCase					:	str=None							,    **UpperCAmelCase					:	Union[str, Any]							,    ):
													# Add extra_ids to the special token list
													if extra_ids > 0 and additional_special_tokens is None:
																				__lowerCamelCase    :						List[Any]						      =   [F"""<extra_id_{i}>""" for i in range(lowerCamelCase__			)]
													elif extra_ids > 0 and additional_special_tokens is not None:
																				# Check that we have the right number of extra_id special tokens
																				__lowerCamelCase    :						List[Any]						      =   len(set(filter(lambda UpperCAmelCase			: bool("extra_id" in str(lowerCamelCase__			)			)							,    lowerCamelCase__			)			)			)
																				if extra_tokens != extra_ids:
																											raise ValueError(
																											    F"""Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"""
																											    " provided to ByT5Tokenizer. In this case the additional_special_tokens must include the"
																											    " extra_ids tokens"			)

													__lowerCamelCase    :						int						      =   AddedToken(lowerCamelCase__							,    lstrip=lowerCamelCase__							,    rstrip=lowerCamelCase__			) if isinstance(lowerCamelCase__							,    lowerCamelCase__			) else pad_token
													__lowerCamelCase    :						List[Any]						      =   AddedToken(lowerCamelCase__							,    lstrip=lowerCamelCase__							,    rstrip=lowerCamelCase__			) if isinstance(lowerCamelCase__							,    lowerCamelCase__			) else eos_token
													__lowerCamelCase    :						Any						      =   AddedToken(lowerCamelCase__							,    lstrip=lowerCamelCase__							,    rstrip=lowerCamelCase__			) if isinstance(lowerCamelCase__							,    lowerCamelCase__			) else unk_token

													super().__init__(
													    eos_token=lowerCamelCase__							,    unk_token=lowerCamelCase__							,    pad_token=lowerCamelCase__							,    extra_ids=lowerCamelCase__							,    additional_special_tokens=lowerCamelCase__							,    **lowerCamelCase__							,    )

													__lowerCamelCase    :						List[str]						      =   extra_ids

													__lowerCamelCase    :						Union[str, Any]						      =   2**8  # utf is 8 bits

													# define special tokens dict
													__lowerCamelCase    :						Optional[int]						      =   {
													    self.pad_token: 0,
													    self.eos_token: 1,
													    self.unk_token: 2,
													}
													__lowerCamelCase    :						Union[str, Any]						      =   len(self.special_tokens_encoder			)
													__lowerCamelCase    :						Dict						      =   len(lowerCamelCase__			)
													for i, token in enumerate(lowerCamelCase__			):
																				__lowerCamelCase    :						Union[str, Any]						      =   self.vocab_size + i - n
													__lowerCamelCase    :						Tuple						      =   {v: k for k, v in self.special_tokens_encoder.items()}






						@property
						def 					lowerCamelCase__   (				self					:	Union[str, Any]			):
													return self._utf_vocab_size + self._num_special_tokens + self._extra_ids






						def 					lowerCamelCase__   (				self					:	Optional[Any]							,    UpperCAmelCase					:	List[int]							,    UpperCAmelCase					:	Optional[List[int]] = None							,    UpperCAmelCase					:	bool = False			):
													if already_has_special_tokens:
																				return super().get_special_tokens_mask(
																				    token_ids_a=lowerCamelCase__							,    token_ids_a=lowerCamelCase__							,    already_has_special_tokens=lowerCamelCase__			)

													# normal case: some special tokens
													if token_ids_a is None:
																				return ([0] * len(lowerCamelCase__			)) + [1]
													return ([0] * len(lowerCamelCase__			)) + [1] + ([0] * len(lowerCamelCase__			)) + [1]






						def 					lowerCamelCase__   (				self					:	str							,    UpperCAmelCase					:	List[int]			):
													if len(lowerCamelCase__			) > 0 and token_ids[-1] == self.eos_token_id:
																				warnings.warn(
																				    F"""This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated"""
																				    " eos tokens being added."			)
																				return token_ids
													else:
																				return token_ids + [self.eos_token_id]






						def 					lowerCamelCase__   (				self					:	str							,    UpperCAmelCase					:	List[int]							,    UpperCAmelCase					:	Optional[List[int]] = None			):
													__lowerCamelCase    :						List[Any]						      =   [self.eos_token_id]

													if token_ids_a is None:
																				return len(token_ids_a + eos			) * [0]
													return len(token_ids_a + eos + token_ids_a + eos			) * [0]






						def 					lowerCamelCase__   (				self					:	Dict							,    UpperCAmelCase					:	List[int]							,    UpperCAmelCase					:	Optional[List[int]] = None			):
													__lowerCamelCase    :						int						      =   self._add_eos_if_not_present(lowerCamelCase__			)
													if token_ids_a is None:
																				return token_ids_a
													else:
																				__lowerCamelCase    :						Any						      =   self._add_eos_if_not_present(lowerCamelCase__			)
																				return token_ids_a + token_ids_a






						def 					lowerCamelCase__   (				self					:	Optional[Any]							,    UpperCAmelCase					:	str			):
													__lowerCamelCase    :						Union[str, Any]						      =   [chr(lowerCamelCase__			) for i in text.encode("utf-8"			)]
													return tokens






						def 					lowerCamelCase__   (				self					:	List[Any]							,    UpperCAmelCase					:	str			):
													if token in self.special_tokens_encoder:
																				__lowerCamelCase    :						str						      =   self.special_tokens_encoder[token]
													elif token in self.added_tokens_encoder:
																				__lowerCamelCase    :						List[Any]						      =   self.added_tokens_encoder[token]
													elif len(lowerCamelCase__			) != 1:
																				__lowerCamelCase    :						Tuple						      =   self.unk_token_id
													else:
																				__lowerCamelCase    :						Tuple						      =   ord(lowerCamelCase__			) + self._num_special_tokens
													return token_id






						def 					lowerCamelCase__   (				self					:	Union[str, Any]							,    UpperCAmelCase					:	Any			):
													if index in self.special_tokens_decoder:
																				__lowerCamelCase    :						List[Any]						      =   self.special_tokens_decoder[index]
													else:
																				__lowerCamelCase    :						str						      =   chr(index - self._num_special_tokens			)
													return token






						def 					lowerCamelCase__   (				self					:	Optional[Any]							,    UpperCAmelCase					:	Union[str, Any]			):
													__lowerCamelCase    :						Any						      =   b""
													for token in tokens:
																				if token in self.special_tokens_decoder:
																											__lowerCamelCase    :						Optional[int]						      =   self.special_tokens_decoder[token].encode("utf-8"			)
																				elif token in self.added_tokens_decoder:
																											__lowerCamelCase    :						List[Any]						      =   self.special_tokens_decoder[token].encode("utf-8"			)
																				elif token in self.special_tokens_encoder:
																											__lowerCamelCase    :						Optional[int]						      =   token.encode("utf-8"			)
																				elif token in self.added_tokens_encoder:
																											__lowerCamelCase    :						Tuple						      =   token.encode("utf-8"			)
																				else:
																											__lowerCamelCase    :						Optional[int]						      =   bytes([ord(lowerCamelCase__			)]			)
																				bstring += tok_string
													__lowerCamelCase    :						Dict						      =   bstring.decode("utf-8"							,    errors="ignore"			)
													return string







						def 					lowerCamelCase__   (				self					:	str							,    UpperCAmelCase					:	str							,    UpperCAmelCase					:	Optional[str] = None			):
													return ()
"""simple docstring"""






from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCAmelCase__  :  str						     =	{
    'configuration_mctct': ['MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'MCTCTConfig'],
    'feature_extraction_mctct': ['MCTCTFeatureExtractor'],
    'processing_mctct': ['MCTCTProcessor'],
}


try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCAmelCase__  :  Any						     =	[
						    'MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST',
						    'MCTCTForCTC',
						    'MCTCTModel',
						    'MCTCTPreTrainedModel',
						]


if TYPE_CHECKING:
						from .configuration_mctct import MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP, MCTCTConfig
						from .feature_extraction_mctct import MCTCTFeatureExtractor
						from .processing_mctct import MCTCTProcessor

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_mctct import MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST, MCTCTForCTC, MCTCTModel, MCTCTPreTrainedModel

else:
						import sys

						lowerCAmelCase__  :  Any						     =	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



