

import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a__				=		logging.get_logger(__name__)

a__				=		{'''vocab_file''': '''spiece.model'''}

a__				=		{
    '''vocab_file''': {
        '''google/bigbird-roberta-base''': '''https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model''',
        '''google/bigbird-roberta-large''': (
            '''https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'''
        ),
        '''google/bigbird-base-trivia-itc''': (
            '''https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'''
        ),
    }
}

a__				=		{
    '''google/bigbird-roberta-base''': 4096,
    '''google/bigbird-roberta-large''': 4096,
    '''google/bigbird-base-trivia-itc''': 4096,
}

class 				UpperCAmelCase_		(			__a   ):
							"""simple docstring"""







							UpperCAmelCase__       :						Tuple	    =    VOCAB_FILES_NAMES
							UpperCAmelCase__       :						List[Any]	    =    PRETRAINED_VOCAB_FILES_MAP
							UpperCAmelCase__       :						Any	    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCAmelCase__       :						Union[str, Any]	    =    ['''input_ids''', '''attention_mask''']
							UpperCAmelCase__       :						List[int]	    =    []


							def __init__(			self       ,							_a       ,							_a="<unk>"       ,							_a="<s>"       ,							_a="</s>"       ,							_a="<pad>"       ,							_a="[SEP]"       ,							_a="[MASK]"       ,							_a="[CLS]"       ,							_a = None       ,							**_a       ,							)    ->    List[str]:
												_a	:    Any         =		AddedToken(lowerCAmelCase__       ,							lstrip=lowerCAmelCase__       ,							rstrip=lowerCAmelCase__       ) if isinstance(lowerCAmelCase__       ,							lowerCAmelCase__       ) else bos_token
												_a	:    Tuple         =		AddedToken(lowerCAmelCase__       ,							lstrip=lowerCAmelCase__       ,							rstrip=lowerCAmelCase__       ) if isinstance(lowerCAmelCase__       ,							lowerCAmelCase__       ) else eos_token
												_a	:    Optional[Any]         =		AddedToken(lowerCAmelCase__       ,							lstrip=lowerCAmelCase__       ,							rstrip=lowerCAmelCase__       ) if isinstance(lowerCAmelCase__       ,							lowerCAmelCase__       ) else unk_token
												_a	:    Dict         =		AddedToken(lowerCAmelCase__       ,							lstrip=lowerCAmelCase__       ,							rstrip=lowerCAmelCase__       ) if isinstance(lowerCAmelCase__       ,							lowerCAmelCase__       ) else pad_token
												_a	:    List[Any]         =		AddedToken(lowerCAmelCase__       ,							lstrip=lowerCAmelCase__       ,							rstrip=lowerCAmelCase__       ) if isinstance(lowerCAmelCase__       ,							lowerCAmelCase__       ) else cls_token
												_a	:    Tuple         =		AddedToken(lowerCAmelCase__       ,							lstrip=lowerCAmelCase__       ,							rstrip=lowerCAmelCase__       ) if isinstance(lowerCAmelCase__       ,							lowerCAmelCase__       ) else sep_token

												# Mask token behave like a normal word, i.e. include the space before it
												_a	:    List[Any]         =		AddedToken(lowerCAmelCase__       ,							lstrip=lowerCAmelCase__       ,							rstrip=lowerCAmelCase__       ) if isinstance(lowerCAmelCase__       ,							lowerCAmelCase__       ) else mask_token

												_a	:    Any         =		{} if sp_model_kwargs is None else sp_model_kwargs

												super().__init__(
												    bos_token=lowerCAmelCase__       ,							eos_token=lowerCAmelCase__       ,							unk_token=lowerCAmelCase__       ,							pad_token=lowerCAmelCase__       ,							sep_token=lowerCAmelCase__       ,							mask_token=lowerCAmelCase__       ,							cls_token=lowerCAmelCase__       ,							sp_model_kwargs=self.sp_model_kwargs       ,							**lowerCAmelCase__       ,							)

												_a	:    Tuple         =		vocab_file

												_a	:    Union[str, Any]         =		spm.SentencePieceProcessor(**self.sp_model_kwargs       )
												self.sp_model.Load(lowerCAmelCase__       )


							@property
							def 				__lowercase				(			self       )    ->    Union[str, Any]:
												return self.sp_model.get_piece_size()


							def 				__lowercase				(			self       )    ->    int:
												_a	:    List[str]         =		{self.convert_ids_to_tokens(lowerCAmelCase__       ): i for i in range(self.vocab_size       )}
												vocab.update(self.added_tokens_encoder       )
												return vocab


							def __getstate__(			self       )    ->    str:
												_a	:    Union[str, Any]         =		self.__dict__.copy()
												_a	:    Dict         =		None
												return state


							def __setstate__(			self       ,							_a       )    ->    Any:
												_a	:    str         =		d

												# for backward compatibility
												if not hasattr(self       ,							'''sp_model_kwargs'''       ):
																	_a	:    str         =		{}

												_a	:    List[Any]         =		spm.SentencePieceProcessor(**self.sp_model_kwargs       )
												self.sp_model.Load(self.vocab_file       )


							def 				__lowercase				(			self       ,							_a       )    ->    Union[str, Any]:
												return self.sp_model.encode(lowerCAmelCase__       ,							out_type=lowerCAmelCase__       )


							def 				__lowercase				(			self       ,							_a       )    ->    int:
												return self.sp_model.piece_to_id(lowerCAmelCase__       )


							def 				__lowercase				(			self       ,							_a       )    ->    Tuple:
												_a	:    Union[str, Any]         =		self.sp_model.IdToPiece(lowerCAmelCase__       )
												return token


							def 				__lowercase				(			self       ,							_a       )    ->    str:
												_a	:    Optional[Any]         =		[]
												_a	:    str         =		''''''
												_a	:    Optional[Any]         =		False
												for token in tokens:
																	# make sure that special tokens are not decoded using sentencepiece model
																	if token in self.all_special_tokens:
																						if not prev_is_special:
																											out_string += " "
																						out_string += self.sp_model.decode(lowerCAmelCase__       ) + token
																						_a	:    Union[str, Any]         =		True
																						_a	:    Optional[Any]         =		[]
																	else:
																						current_sub_tokens.append(lowerCAmelCase__       )
																						_a	:    List[str]         =		False
												out_string += self.sp_model.decode(lowerCAmelCase__       )
												return out_string.strip()


							def 				__lowercase				(			self       ,							_a       ,							_a = False       ,							_a = None       ,							_a = True       ,							**_a       ,							)    ->    str:
												_a	:    Tuple         =		kwargs.pop('''use_source_tokenizer'''       ,							lowerCAmelCase__       )

												_a	:    int         =		self.convert_ids_to_tokens(lowerCAmelCase__       ,							skip_special_tokens=lowerCAmelCase__       )

												# To avoid mixing byte-level and unicode for byte-level BPT
												# we need to build string separately for added tokens and byte-level tokens
												# cf. https://github.com/huggingface/transformers/issues/1133
												_a	:    List[Any]         =		[]
												_a	:    Optional[int]         =		[]
												for token in filtered_tokens:
																	if skip_special_tokens and token in self.all_special_ids:
																						continue
																	if token in self.added_tokens_encoder:
																						if current_sub_text:
																											sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__       )       )
																											_a	:    Dict         =		[]
																						sub_texts.append(lowerCAmelCase__       )
																	else:
																						current_sub_text.append(lowerCAmelCase__       )
												if current_sub_text:
																	sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__       )       )

												# Mimic the behavior of the Rust tokenizer:
												# No space before [MASK] and [SEP]
												if spaces_between_special_tokens:
																	_a	:    int         =		re.sub(R''' (\[(MASK|SEP)\])'''       ,							R'''\1'''       ,							''' '''.join(lowerCAmelCase__       )       )
												else:
																	_a	:    Dict         =		''''''.join(lowerCAmelCase__       )

												_a	:    Any         =		(
												    clean_up_tokenization_spaces
												    if clean_up_tokenization_spaces is not None
												    else self.clean_up_tokenization_spaces
												)
												if clean_up_tokenization_spaces:
																	_a	:    Union[str, Any]         =		self.clean_up_tokenization(lowerCAmelCase__       )
																	return clean_text
												else:
																	return text


							def 				__lowercase				(			self       ,							_a       ,							_a = None       )    ->    Union[str, Any]:
												if not os.path.isdir(lowerCAmelCase__       ):
																	logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
																	return
												_a	:    Optional[int]         =		os.path.join(
												    lowerCAmelCase__       ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )

												if os.path.abspath(self.vocab_file       ) != os.path.abspath(lowerCAmelCase__       ) and os.path.isfile(self.vocab_file       ):
																	copyfile(self.vocab_file       ,							lowerCAmelCase__       )
												elif not os.path.isfile(self.vocab_file       ):
																	with open(lowerCAmelCase__       ,							'''wb'''       ) as fi:
																						_a	:    Optional[int]         =		self.sp_model.serialized_model_proto()
																						fi.write(lowerCAmelCase__       )

												return (out_vocab_file,)


							def 				__lowercase				(			self       ,							_a       ,							_a = None       )    ->    List[str]:
												if token_ids_a is None:
																	return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
												_a	:    Optional[Any]         =		[self.cls_token_id]
												_a	:    int         =		[self.sep_token_id]
												return cls + token_ids_a + sep + token_ids_a + sep


							def 				__lowercase				(			self       ,							_a       ,							_a = None       ,							_a = False       )    ->    Any:
												if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=lowerCAmelCase__       ,							token_ids_a=lowerCAmelCase__       ,							already_has_special_tokens=lowerCAmelCase__       )

												if token_ids_a is None:
																	return [1] + ([0] * len(lowerCAmelCase__       )) + [1]
												return [1] + ([0] * len(lowerCAmelCase__       )) + [1] + ([0] * len(lowerCAmelCase__       )) + [1]




							def 				__lowercase				(			self       ,							_a       ,							_a = None       )    ->    Optional[Any]:
												_a	:    str         =		[self.sep_token_id]
												_a	:    Dict         =		[self.cls_token_id]
												if token_ids_a is None:
																	return len(cls + token_ids_a + sep       ) * [0]
												return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]





"""simple docstring"""





from typing import List, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__magic_name__       =			logging.get_logger(__name__)

__magic_name__       =			{
    "huggingface/informer-tourism-monthly": (
        "https://huggingface.co/huggingface/informer-tourism-monthly/resolve/main/config.json"
    ),
    # See all Informer models at https://huggingface.co/models?filter=informer
}
class 						SCREAMING_SNAKE_CASE_ (						__a ):




		"""simple docstring"""





		__lowercase							:  Dict						   =		'''informer'''
		__lowercase							:  Union[str, Any]						   =		{
		    '''hidden_size''': '''d_model''',
		    '''num_attention_heads''': '''encoder_attention_heads''',
		    '''num_hidden_layers''': '''encoder_layers''',
		}




		def __init__(     self							,  lowerCAmelCase__ = None							,  lowerCAmelCase__ = None							,  lowerCAmelCase__ = "student_t"							,  lowerCAmelCase__ = "nll"							,  lowerCAmelCase__ = 1							,  lowerCAmelCase__ = None							,  lowerCAmelCase__ = "mean"							,  lowerCAmelCase__ = 0							,  lowerCAmelCase__ = 0							,  lowerCAmelCase__ = 0							,  lowerCAmelCase__ = 0							,  lowerCAmelCase__ = None							,  lowerCAmelCase__ = None							,  lowerCAmelCase__ = 6_4							,  lowerCAmelCase__ = 3_2							,  lowerCAmelCase__ = 3_2							,  lowerCAmelCase__ = 2							,  lowerCAmelCase__ = 2							,  lowerCAmelCase__ = 2							,  lowerCAmelCase__ = 2							,  lowerCAmelCase__ = True							,  lowerCAmelCase__ = "gelu"							,  lowerCAmelCase__ = 0.05							,  lowerCAmelCase__ = 0.1							,  lowerCAmelCase__ = 0.1							,  lowerCAmelCase__ = 0.1							,  lowerCAmelCase__ = 0.1							,  lowerCAmelCase__ = 1_0_0							,  lowerCAmelCase__ = 0.02							,  lowerCAmelCase__=True							,  lowerCAmelCase__ = "prob"							,  lowerCAmelCase__ = 5							,  lowerCAmelCase__ = True							,  **lowerCAmelCase__							,  ):
				# time series specific configuration
				__SCREAMING_SNAKE_CASE            = prediction_length
				__SCREAMING_SNAKE_CASE            = context_length or prediction_length
				__SCREAMING_SNAKE_CASE            = distribution_output
				__SCREAMING_SNAKE_CASE            = loss
				__SCREAMING_SNAKE_CASE            = input_size
				__SCREAMING_SNAKE_CASE            = num_time_features
				__SCREAMING_SNAKE_CASE            = lags_sequence if lags_sequence is not None else [1, 2, 3, 4, 5, 6, 7]
				__SCREAMING_SNAKE_CASE            = scaling
				__SCREAMING_SNAKE_CASE            = num_dynamic_real_features
				__SCREAMING_SNAKE_CASE            = num_static_real_features
				__SCREAMING_SNAKE_CASE            = num_static_categorical_features

				# set cardinality
				if cardinality and num_static_categorical_features > 0:
						if len(lowerCAmelCase__) != num_static_categorical_features:
								raise ValueError(
								    """The cardinality should be a list of the same length as `num_static_categorical_features`""")
						__SCREAMING_SNAKE_CASE            = cardinality
				else:
						__SCREAMING_SNAKE_CASE            = [0]

				# set embedding_dimension
				if embedding_dimension and num_static_categorical_features > 0:
						if len(lowerCAmelCase__) != num_static_categorical_features:
								raise ValueError(
								    """The embedding dimension should be a list of the same length as `num_static_categorical_features`""")
						__SCREAMING_SNAKE_CASE            = embedding_dimension
				else:
						__SCREAMING_SNAKE_CASE            = [min(5_0							,  (cat + 1) // 2) for cat in self.cardinality]

				__SCREAMING_SNAKE_CASE            = num_parallel_samples

				# Transformer architecture configuration
				__SCREAMING_SNAKE_CASE            = input_size * len(self.lags_sequence) + self._number_of_features
				__SCREAMING_SNAKE_CASE            = d_model
				__SCREAMING_SNAKE_CASE            = encoder_attention_heads
				__SCREAMING_SNAKE_CASE            = decoder_attention_heads
				__SCREAMING_SNAKE_CASE            = encoder_ffn_dim
				__SCREAMING_SNAKE_CASE            = decoder_ffn_dim
				__SCREAMING_SNAKE_CASE            = encoder_layers
				__SCREAMING_SNAKE_CASE            = decoder_layers

				__SCREAMING_SNAKE_CASE            = dropout
				__SCREAMING_SNAKE_CASE            = attention_dropout
				__SCREAMING_SNAKE_CASE            = activation_dropout
				__SCREAMING_SNAKE_CASE            = encoder_layerdrop
				__SCREAMING_SNAKE_CASE            = decoder_layerdrop

				__SCREAMING_SNAKE_CASE            = activation_function
				__SCREAMING_SNAKE_CASE            = init_std

				__SCREAMING_SNAKE_CASE            = use_cache

				# Informer
				__SCREAMING_SNAKE_CASE            = attention_type
				__SCREAMING_SNAKE_CASE            = sampling_factor
				__SCREAMING_SNAKE_CASE            = distil

				super().__init__(is_encoder_decoder=lowerCAmelCase__							,  **lowerCAmelCase__)




		@property
		def 	snake_case_     (     self):
				return (
				    sum(self.embedding_dimension)
				    + self.num_dynamic_real_features
				    + self.num_time_features
				    + self.num_static_real_features
				    + self.input_size * 2  # the log1p(abs(loc)) and log(scale) features
				)




