






"""simple docstring"""
import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase_					=      logging.get_logger(__name__)

lowerCamelCase_					=      {
    '''asapp/sew-d-tiny-100k''': '''https://huggingface.co/asapp/sew-d-tiny-100k/resolve/main/config.json''',
    # See all SEW-D models at https://huggingface.co/models?filter=sew-d
}







class 			UpperCamelCase_       (__A     ):
	__magic_name__	      =				'''sew-d'''





	def __init__(      self					: Optional[Any]	,	lowerCAmelCase_					: List[str]=32	,	lowerCAmelCase_					: Dict=768	,	lowerCAmelCase_					: List[str]=12	,	lowerCAmelCase_					: List[Any]=12	,	lowerCAmelCase_					: Tuple=3_072	,	lowerCAmelCase_					: Any=2	,	lowerCAmelCase_					: Optional[Any]=512	,	lowerCAmelCase_					: List[str]=256	,	lowerCAmelCase_					: int=True	,	lowerCAmelCase_					: Dict=True	,	lowerCAmelCase_					: str=("p2c", "c2p")	,	lowerCAmelCase_					: Union[str, Any]="layer_norm"	,	lowerCAmelCase_					: Union[str, Any]="gelu_python"	,	lowerCAmelCase_					: Optional[Any]=0.1	,	lowerCAmelCase_					: int=0.1	,	lowerCAmelCase_					: Any=0.1	,	lowerCAmelCase_					: int=0.0	,	lowerCAmelCase_					: Optional[int]=0.1	,	lowerCAmelCase_					: List[Any]=0.0_2	,	lowerCAmelCase_					: Union[str, Any]=1e-7	,	lowerCAmelCase_					: List[str]=1e-5	,	lowerCAmelCase_					: Dict="group"	,	lowerCAmelCase_					: Dict="gelu"	,	lowerCAmelCase_					: Optional[int]=(64, 128, 128, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512)	,	lowerCAmelCase_					: Tuple=(5, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1)	,	lowerCAmelCase_					: Union[str, Any]=(10, 3, 1, 3, 1, 3, 1, 3, 1, 2, 1, 2, 1)	,	lowerCAmelCase_					: str=False	,	lowerCAmelCase_					: Optional[Any]=128	,	lowerCAmelCase_					: Optional[int]=16	,	lowerCAmelCase_					: Optional[int]=True	,	lowerCAmelCase_					: int=0.0_5	,	lowerCAmelCase_					: Any=10	,	lowerCAmelCase_					: Dict=2	,	lowerCAmelCase_					: List[Any]=0.0	,	lowerCAmelCase_					: Union[str, Any]=10	,	lowerCAmelCase_					: Dict=0	,	lowerCAmelCase_					: Optional[Any]="mean"	,	lowerCAmelCase_					: Tuple=False	,	lowerCAmelCase_					: List[str]=False	,	lowerCAmelCase_					: List[str]=256	,	lowerCAmelCase_					: Optional[Any]=0	,	lowerCAmelCase_					: int=1	,	lowerCAmelCase_					: Union[str, Any]=2	,	**lowerCAmelCase_					: Optional[int]	,	)						->			Dict:
				super().__init__(**lowerCAmelCase_	,	pad_token_id=lowerCAmelCase_	,	bos_token_id=lowerCAmelCase_	,	eos_token_id=lowerCAmelCase_						)
				UpperCAmelCase_     : List[str]			     =					hidden_size
				UpperCAmelCase_     : Tuple			     =					feat_extract_norm
				UpperCAmelCase_     : Optional[Any]			     =					feat_extract_activation
				UpperCAmelCase_     : str			     =					list(lowerCAmelCase_						)
				UpperCAmelCase_     : Any			     =					list(lowerCAmelCase_						)
				UpperCAmelCase_     : Optional[Any]			     =					list(lowerCAmelCase_						)
				UpperCAmelCase_     : Tuple			     =					conv_bias
				UpperCAmelCase_     : List[str]			     =					num_conv_pos_embeddings
				UpperCAmelCase_     : Tuple			     =					num_conv_pos_embedding_groups
				UpperCAmelCase_     : Tuple			     =					len(self.conv_dim						)
				UpperCAmelCase_     : Optional[Any]			     =					num_hidden_layers
				UpperCAmelCase_     : Any			     =					intermediate_size
				UpperCAmelCase_     : Tuple			     =					squeeze_factor
				UpperCAmelCase_     : Optional[Any]			     =					max_position_embeddings
				UpperCAmelCase_     : Tuple			     =					position_buckets
				UpperCAmelCase_     : List[str]			     =					share_att_key
				UpperCAmelCase_     : Dict			     =					relative_attention
				UpperCAmelCase_     : Union[str, Any]			     =					norm_rel_ebd
				UpperCAmelCase_     : Dict			     =					list(lowerCAmelCase_						)
				UpperCAmelCase_     : Optional[Any]			     =					hidden_act
				UpperCAmelCase_     : List[Any]			     =					num_attention_heads
				UpperCAmelCase_     : Optional[int]			     =					hidden_dropout
				UpperCAmelCase_     : Union[str, Any]			     =					attention_dropout
				UpperCAmelCase_     : int			     =					activation_dropout
				UpperCAmelCase_     : Tuple			     =					feat_proj_dropout
				UpperCAmelCase_     : Optional[int]			     =					final_dropout
				UpperCAmelCase_     : Tuple			     =					layer_norm_eps
				UpperCAmelCase_     : str			     =					feature_layer_norm_eps
				UpperCAmelCase_     : Union[str, Any]			     =					initializer_range
				UpperCAmelCase_     : Any			     =					vocab_size

				if (
				    (len(self.conv_stride						) != self.num_feat_extract_layers)
				    or (len(self.conv_kernel						) != self.num_feat_extract_layers)
				    or (len(self.conv_dim						) != self.num_feat_extract_layers)
				):
							raise ValueError(
							    "Configuration for convolutional layers is incorrect."
							    "It is required that `len(config.conv_dim)` == `len(config.conv_stride)` == `len(config.conv_kernel)`,"
							    f"""but is `len(config.conv_dim) = {len(self.conv_dim						)}`, `len(config.conv_stride)"""
							    f"""= {len(self.conv_stride						)}`, `len(config.conv_kernel) = {len(self.conv_kernel						)}`."""						)

				# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
				UpperCAmelCase_     : Optional[int]			     =					apply_spec_augment
				UpperCAmelCase_     : Optional[Any]			     =					mask_time_prob
				UpperCAmelCase_     : int			     =					mask_time_length
				UpperCAmelCase_     : List[Any]			     =					mask_time_min_masks
				UpperCAmelCase_     : Optional[Any]			     =					mask_feature_prob
				UpperCAmelCase_     : List[Any]			     =					mask_feature_length
				UpperCAmelCase_     : str			     =					mask_feature_min_masks

				# ctc loss
				UpperCAmelCase_     : Any			     =					ctc_loss_reduction
				UpperCAmelCase_     : Dict			     =					ctc_zero_infinity

				# sequence classification
				UpperCAmelCase_     : Dict			     =					use_weighted_layer_sum
				UpperCAmelCase_     : Union[str, Any]			     =					classifier_proj_size





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple						)						->			Optional[Any]:
				return functools.reduce(operator.mul	,	self.conv_stride	,	1						)











"""simple docstring"""
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
			from .tokenization_mbart import MBartTokenizer
else:
			lowerCamelCase_					=      None


lowerCamelCase_					=      logging.get_logger(__name__)


lowerCamelCase_					=      {'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase_					=      {
    '''vocab_file''': {
        '''facebook/mbart-large-en-ro''': (
            '''https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'''
        ),
        '''facebook/mbart-large-cc25''': (
            '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/mbart-large-en-ro''': '''https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json''',
        '''facebook/mbart-large-cc25''': '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json''',
    },
}

lowerCamelCase_					=      {
    '''facebook/mbart-large-en-ro''': 1024,
    '''facebook/mbart-large-cc25''': 1024,
}

# fmt: off
lowerCamelCase_					=      ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''']







class 			UpperCamelCase_       (__A     ):
	__magic_name__	      =				VOCAB_FILES_NAMES
	__magic_name__	      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	__magic_name__	      =				PRETRAINED_VOCAB_FILES_MAP
	__magic_name__	      =				['''input_ids''', '''attention_mask''']
	__magic_name__	      =				MBartTokenizer

	__magic_name__	      =				[]
	__magic_name__	      =				[]





	def __init__(      self					: List[str]	,	lowerCAmelCase_					: List[Any]=None	,	lowerCAmelCase_					: str=None	,	lowerCAmelCase_					: List[Any]="<s>"	,	lowerCAmelCase_					: Optional[Any]="</s>"	,	lowerCAmelCase_					: str="</s>"	,	lowerCAmelCase_					: str="<s>"	,	lowerCAmelCase_					: List[Any]="<unk>"	,	lowerCAmelCase_					: Tuple="<pad>"	,	lowerCAmelCase_					: Union[str, Any]="<mask>"	,	lowerCAmelCase_					: List[Any]=None	,	lowerCAmelCase_					: str=None	,	lowerCAmelCase_					: Optional[Any]=None	,	**lowerCAmelCase_					: Any	,	)						->			Tuple:
				# Mask token behave like a normal word, i.e. include the space before it
				UpperCAmelCase_     : Union[str, Any]			     =					AddedToken(lowerCAmelCase_	,	lstrip=lowerCAmelCase_	,	rstrip=lowerCAmelCase_						) if isinstance(lowerCAmelCase_	,	lowerCAmelCase_						) else mask_token

				super().__init__(
				    vocab_file=lowerCAmelCase_	,	tokenizer_file=lowerCAmelCase_	,	bos_token=lowerCAmelCase_	,	eos_token=lowerCAmelCase_	,	sep_token=lowerCAmelCase_	,	cls_token=lowerCAmelCase_	,	unk_token=lowerCAmelCase_	,	pad_token=lowerCAmelCase_	,	mask_token=lowerCAmelCase_	,	src_lang=lowerCAmelCase_	,	tgt_lang=lowerCAmelCase_	,	additional_special_tokens=lowerCAmelCase_	,	**lowerCAmelCase_	,	)

				UpperCAmelCase_     : Tuple			     =					vocab_file
				UpperCAmelCase_     : str			     =					False if not self.vocab_file else True

				UpperCAmelCase_     : List[Any]			     =					FAIRSEQ_LANGUAGE_CODES.copy()

				if additional_special_tokens is not None:
							# Only add those special tokens if they are not already there.
							_additional_special_tokens.extend(
							    [t for t in additional_special_tokens if t not in _additional_special_tokens]						)

				self.add_special_tokens({"additional_special_tokens": _additional_special_tokens}						)
				UpperCAmelCase_     : Tuple			     =					{
				    lang_code: self.convert_tokens_to_ids(lowerCAmelCase_						) for lang_code in FAIRSEQ_LANGUAGE_CODES
				}

				UpperCAmelCase_     : int			     =					src_lang if src_lang is not None else "en_XX"
				UpperCAmelCase_     : Union[str, Any]			     =					self.convert_tokens_to_ids(self._src_lang						)
				UpperCAmelCase_     : Any			     =					tgt_lang
				self.set_src_lang_special_tokens(self._src_lang						)





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[int]						)						->			str:
				return self._src_lang





	@src_lang.setter
	def      _SCREAMING_SNAKE_CASE      (      self					: Any	,	lowerCAmelCase_					: str						)						->			None:
				UpperCAmelCase_     : List[Any]			     =					new_src_lang
				self.set_src_lang_special_tokens(self._src_lang						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]	,	lowerCAmelCase_					: List[int]	,	lowerCAmelCase_					: Optional[List[int]] = None						)						->			List[int]:
				if token_ids_a is None:
							return self.prefix_tokens + token_ids_a + self.suffix_tokens
				# We don't expect to process pairs, but leave the pair logic for API consistency
				return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens





	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]	,	lowerCAmelCase_					: List[int]	,	lowerCAmelCase_					: Optional[List[int]] = None						)						->			List[int]:
				UpperCAmelCase_     : List[str]			     =					[self.sep_token_id]
				UpperCAmelCase_     : Dict			     =					[self.cls_token_id]

				if token_ids_a is None:
							return len(cls + token_ids_a + sep						) * [0]
				return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]





	def      _SCREAMING_SNAKE_CASE      (      self					: Any	,	lowerCAmelCase_					: Tuple	,	lowerCAmelCase_					: str	,	lowerCAmelCase_					: Optional[str]	,	lowerCAmelCase_					: Optional[str]	,	**lowerCAmelCase_					: Union[str, Any]						)						->			Dict:
				if src_lang is None or tgt_lang is None:
							raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"						)
				UpperCAmelCase_     : str			     =					src_lang
				UpperCAmelCase_     : str			     =					self(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_	,	return_tensors=lowerCAmelCase_	,	**lowerCAmelCase_						)
				UpperCAmelCase_     : Optional[int]			     =					self.convert_tokens_to_ids(lowerCAmelCase_						)
				UpperCAmelCase_     : Dict			     =					tgt_lang_id
				return inputs





	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]	,	lowerCAmelCase_					: List[str]	,	lowerCAmelCase_					: str = "en_XX"	,	lowerCAmelCase_					: Optional[List[str]] = None	,	lowerCAmelCase_					: str = "ro_RO"	,	**lowerCAmelCase_					: Dict	,	)						->			BatchEncoding:
				UpperCAmelCase_     : List[Any]			     =					src_lang
				UpperCAmelCase_     : Tuple			     =					tgt_lang
				return super().prepare_seqaseq_batch(lowerCAmelCase_	,	lowerCAmelCase_	,	**lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[int]						)						->			Any:
				return self.set_src_lang_special_tokens(self.src_lang						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]						)						->			Dict:
				return self.set_tgt_lang_special_tokens(self.tgt_lang						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]	,	lowerCAmelCase_					: List[Any]						)						->			None:
				UpperCAmelCase_     : int			     =					self.convert_tokens_to_ids(lowerCAmelCase_						)
				UpperCAmelCase_     : str			     =					[]
				UpperCAmelCase_     : str			     =					[self.eos_token_id, self.cur_lang_code]

				UpperCAmelCase_     : Union[str, Any]			     =					self.convert_ids_to_tokens(self.prefix_tokens						)
				UpperCAmelCase_     : List[Any]			     =					self.convert_ids_to_tokens(self.suffix_tokens						)

				UpperCAmelCase_     : Optional[Any]			     =					processors.TemplateProcessing(
				    single=prefix_tokens_str + ["$A"] + suffix_tokens_str	,	pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str	,	special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str	,	self.prefix_tokens + self.suffix_tokens						)						)	,	)





	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple	,	lowerCAmelCase_					: str						)						->			None:
				UpperCAmelCase_     : Union[str, Any]			     =					self.convert_tokens_to_ids(lowerCAmelCase_						)
				UpperCAmelCase_     : Dict			     =					[]
				UpperCAmelCase_     : Optional[int]			     =					[self.eos_token_id, self.cur_lang_code]

				UpperCAmelCase_     : Tuple			     =					self.convert_ids_to_tokens(self.prefix_tokens						)
				UpperCAmelCase_     : Tuple			     =					self.convert_ids_to_tokens(self.suffix_tokens						)

				UpperCAmelCase_     : Optional[Any]			     =					processors.TemplateProcessing(
				    single=prefix_tokens_str + ["$A"] + suffix_tokens_str	,	pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str	,	special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str	,	self.prefix_tokens + self.suffix_tokens						)						)	,	)





	def      _SCREAMING_SNAKE_CASE      (      self					: str	,	lowerCAmelCase_					: str	,	lowerCAmelCase_					: Optional[str] = None						)						->			Tuple[str]:
				if not self.can_save_slow_tokenizer:
							raise ValueError(
							    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
							    "tokenizer."						)

				if not os.path.isdir(lowerCAmelCase_						):
							logger.error(f"""Vocabulary path ({save_directory}) should be a directory."""						)
							return
				UpperCAmelCase_     : List[str]			     =					os.path.join(
				    lowerCAmelCase_	,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

				if os.path.abspath(self.vocab_file						) != os.path.abspath(lowerCAmelCase_						):
							copyfile(self.vocab_file	,	lowerCAmelCase_						)

				return (out_vocab_file,)



