






"""simple docstring"""
import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__    						=			logging.get_logger(__name__)


lowerCAmelCase__    						=			{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

lowerCAmelCase__    						=			{
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
}

lowerCAmelCase__    						=			{
    'allenai/longformer-base-4096': 4_096,
    'allenai/longformer-large-4096': 4_096,
    'allenai/longformer-large-4096-finetuned-triviaqa': 4_096,
    'allenai/longformer-base-4096-extra.pos.embd.only': 4_096,
    'allenai/longformer-large-4096-extra.pos.embd.only': 4_096,
}





@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def 				lowercase__		(					):
				_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    (
				    list(range(ord('!'       ),						ord('~'       ) + 1       )       ) + list(range(ord('¡'       ),						ord('¬'       ) + 1       )       ) + list(range(ord('®'       ),						ord('ÿ'       ) + 1       )       )
				)
				_SCREAMING_SNAKE_CASE   :		Any       		=    bs[:]
				_SCREAMING_SNAKE_CASE   :		Tuple       		=    0
				for b in range(2**8       ):
								if b not in bs:
												bs.append(lowerCamelCase       )
												cs.append(2**8 + n       )
												n += 1
				_SCREAMING_SNAKE_CASE   :		List[str]       		=    [chr(lowerCamelCase       ) for n in cs]
				return dict(zip(lowerCamelCase,						lowerCamelCase       )       )





def 				lowercase__		(					lowerCamelCase       ):
				_SCREAMING_SNAKE_CASE   :		Dict       		=    set()
				_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)       )
								_SCREAMING_SNAKE_CASE   :		str       		=    char
				return pairs






class _lowerCAmelCase						(    __UpperCAmelCase       ):
						SCREAMING_SNAKE_CASE_:					Any 					=				VOCAB_FILES_NAMES
						SCREAMING_SNAKE_CASE_:					Optional[int] 					=				PRETRAINED_VOCAB_FILES_MAP
						SCREAMING_SNAKE_CASE_:					int 					=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						SCREAMING_SNAKE_CASE_:					int 					=				['input_ids', 'attention_mask']







						def __init__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_   ,				lowerCAmelCase_="replace"   ,				lowerCAmelCase_="<s>"   ,				lowerCAmelCase_="</s>"   ,				lowerCAmelCase_="</s>"   ,				lowerCAmelCase_="<s>"   ,				lowerCAmelCase_="<unk>"   ,				lowerCAmelCase_="<pad>"   ,				lowerCAmelCase_="<mask>"   ,				lowerCAmelCase_=False   ,				**lowerCAmelCase_   ,				)		->							int:
										_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    AddedToken(lowerCAmelCase_   ,				lstrip=lowerCAmelCase_   ,				rstrip=lowerCAmelCase_							) if isinstance(lowerCAmelCase_   ,				lowerCAmelCase_							) else bos_token
										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    AddedToken(lowerCAmelCase_   ,				lstrip=lowerCAmelCase_   ,				rstrip=lowerCAmelCase_							) if isinstance(lowerCAmelCase_   ,				lowerCAmelCase_							) else eos_token
										_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    AddedToken(lowerCAmelCase_   ,				lstrip=lowerCAmelCase_   ,				rstrip=lowerCAmelCase_							) if isinstance(lowerCAmelCase_   ,				lowerCAmelCase_							) else sep_token
										_SCREAMING_SNAKE_CASE   :		Dict       		=    AddedToken(lowerCAmelCase_   ,				lstrip=lowerCAmelCase_   ,				rstrip=lowerCAmelCase_							) if isinstance(lowerCAmelCase_   ,				lowerCAmelCase_							) else cls_token
										_SCREAMING_SNAKE_CASE   :		int       		=    AddedToken(lowerCAmelCase_   ,				lstrip=lowerCAmelCase_   ,				rstrip=lowerCAmelCase_							) if isinstance(lowerCAmelCase_   ,				lowerCAmelCase_							) else unk_token
										_SCREAMING_SNAKE_CASE   :		str       		=    AddedToken(lowerCAmelCase_   ,				lstrip=lowerCAmelCase_   ,				rstrip=lowerCAmelCase_							) if isinstance(lowerCAmelCase_   ,				lowerCAmelCase_							) else pad_token

										# Mask token behave like a normal word, i.e. include the space before it
										_SCREAMING_SNAKE_CASE   :		int       		=    AddedToken(lowerCAmelCase_   ,				lstrip=lowerCAmelCase_   ,				rstrip=lowerCAmelCase_							) if isinstance(lowerCAmelCase_   ,				lowerCAmelCase_							) else mask_token

										super().__init__(
										    errors=lowerCAmelCase_   ,				bos_token=lowerCAmelCase_   ,				eos_token=lowerCAmelCase_   ,				unk_token=lowerCAmelCase_   ,				sep_token=lowerCAmelCase_   ,				cls_token=lowerCAmelCase_   ,				pad_token=lowerCAmelCase_   ,				mask_token=lowerCAmelCase_   ,				add_prefix_space=lowerCAmelCase_   ,				**lowerCAmelCase_   ,				)

										with open(lowerCAmelCase_   ,				encoding='utf-8'							) as vocab_handle:
														_SCREAMING_SNAKE_CASE   :		int       		=    json.load(lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		Dict       		=    {v: k for k, v in self.encoder.items()}
										_SCREAMING_SNAKE_CASE   :		Dict       		=    errors  # how to handle errors in decoding
										_SCREAMING_SNAKE_CASE   :		Tuple       		=    bytes_to_unicode()
										_SCREAMING_SNAKE_CASE   :		Dict       		=    {v: k for k, v in self.byte_encoder.items()}
										with open(lowerCAmelCase_   ,				encoding='utf-8'							) as merges_handle:
														_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    merges_handle.read().split('\n'							)[1:-1]
										_SCREAMING_SNAKE_CASE   :		List[str]       		=    [tuple(merge.split()							) for merge in bpe_merges]
										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    dict(zip(lowerCAmelCase_   ,				range(len(lowerCAmelCase_							)							)							)							)
										_SCREAMING_SNAKE_CASE   :		Dict       		=    {}
										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    add_prefix_space

										# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'							)







						@property
						def 			A      (    self							)		->							Union[str, Any]:
										return len(self.encoder							)







						def 			A      (    self							)		->							List[Any]:
										return dict(self.encoder   ,				**self.added_tokens_encoder							)







						def 			A      (    self   ,				lowerCAmelCase_							)		->							Any:
										if token in self.cache:
														return self.cache[token]
										_SCREAMING_SNAKE_CASE   :		str       		=    tuple(lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    get_pairs(lowerCAmelCase_							)

										if not pairs:
														return token

										while True:
														_SCREAMING_SNAKE_CASE   :		int       		=    min(lowerCAmelCase_   ,				key=lambda lowerCAmelCase_							: self.bpe_ranks.get(lowerCAmelCase_   ,				float('inf'							)							)							)
														if bigram not in self.bpe_ranks:
																		break
														_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE   :		str       		=    bigram
														_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    []
														_SCREAMING_SNAKE_CASE   :		Tuple       		=    0
														while i < len(lowerCAmelCase_							):
																		try:
																						_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    word.index(lowerCAmelCase_   ,				lowerCAmelCase_							)
																		except ValueError:
																						new_word.extend(word[i:]							)
																						break
																		else:
																						new_word.extend(word[i:j]							)
																						_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    j

																		if word[i] == first and i < len(lowerCAmelCase_							) - 1 and word[i + 1] == second:
																						new_word.append(first + second							)
																						i += 2
																		else:
																						new_word.append(word[i]							)
																						i += 1
														_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    tuple(lowerCAmelCase_							)
														_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    new_word
														if len(lowerCAmelCase_							) == 1:
																		break
														else:
																		_SCREAMING_SNAKE_CASE   :		int       		=    get_pairs(lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		str       		=    ' '.join(lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		List[str]       		=    word
										return word







						def 			A      (    self   ,				lowerCAmelCase_							)		->							Union[str, Any]:
										_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    []
										for token in re.findall(self.pat   ,				lowerCAmelCase_							):
														_SCREAMING_SNAKE_CASE   :		Dict       		=    ''.join(
														    self.byte_encoder[b] for b in token.encode('utf-8'							)							)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
														bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCAmelCase_							).split(' '							)							)
										return bpe_tokens







						def 			A      (    self   ,				lowerCAmelCase_							)		->							Union[str, Any]:
										return self.encoder.get(lowerCAmelCase_   ,				self.encoder.get(self.unk_token							)							)







						def 			A      (    self   ,				lowerCAmelCase_							)		->							Optional[Any]:
										return self.decoder.get(lowerCAmelCase_							)







						def 			A      (    self   ,				lowerCAmelCase_							)		->							Optional[Any]:
										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    ''.join(lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    bytearray([self.byte_decoder[c] for c in text]							).decode('utf-8'   ,				errors=self.errors							)
										return text







						def 			A      (    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_ = None							)		->							Tuple[str]:
										if not os.path.isdir(lowerCAmelCase_							):
														logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""							)
														return
										_SCREAMING_SNAKE_CASE   :		List[str]       		=    os.path.join(
										    lowerCAmelCase_   ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)
										_SCREAMING_SNAKE_CASE   :		str       		=    os.path.join(
										    lowerCAmelCase_   ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']							)

										with open(lowerCAmelCase_   ,				'w'   ,				encoding='utf-8'							) as f:
														f.write(json.dumps(self.encoder   ,				indent=2   ,				sort_keys=lowerCAmelCase_   ,				ensure_ascii=lowerCAmelCase_							) + '\n'							)

										_SCREAMING_SNAKE_CASE   :		int       		=    0
										with open(lowerCAmelCase_   ,				'w'   ,				encoding='utf-8'							) as writer:
														writer.write('#version: 0.2\n'							)
														for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,				key=lambda lowerCAmelCase_							: kv[1]							):
																		if index != token_index:
																						logger.warning(
																						    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																						    ' Please check that the tokenizer is not corrupted!'							)
																						_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    token_index
																		writer.write(' '.join(lowerCAmelCase_							) + '\n'							)
																		index += 1

										return vocab_file, merge_file







						def 			A      (    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_ = None							)		->							List[int]:
										if token_ids_a is None:
														return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    [self.cls_token_id]
										_SCREAMING_SNAKE_CASE   :		Dict       		=    [self.sep_token_id]
										return cls + token_ids_a + sep + sep + token_ids_a + sep







						def 			A      (    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_ = None   ,				lowerCAmelCase_ = False							)		->							List[int]:
										if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=lowerCAmelCase_   ,				token_ids_a=lowerCAmelCase_   ,				already_has_special_tokens=lowerCAmelCase_							)

										if token_ids_a is None:
														return [1] + ([0] * len(lowerCAmelCase_							)) + [1]
										return [1] + ([0] * len(lowerCAmelCase_							)) + [1, 1] + ([0] * len(lowerCAmelCase_							)) + [1]







						def 			A      (    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_ = None							)		->							List[int]:
										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    [self.sep_token_id]
										_SCREAMING_SNAKE_CASE   :		str       		=    [self.cls_token_id]

										if token_ids_a is None:
														return len(cls + token_ids_a + sep							) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]







						def 			A      (    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_=False   ,				**lowerCAmelCase_							)		->							Any:
										_SCREAMING_SNAKE_CASE   :		str       		=    kwargs.pop('add_prefix_space'   ,				self.add_prefix_space							)
										if (is_split_into_words or add_prefix_space) and (len(lowerCAmelCase_							) > 0 and not text[0].isspace()):
														_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    ' ' + text
										return (text, kwargs)









"""simple docstring"""
from typing import List, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase__    						=			logging.get_logger(__name__)

lowerCAmelCase__    						=			{
    'huggingface/time-series-transformer-tourism-monthly': (
        'https://huggingface.co/huggingface/time-series-transformer-tourism-monthly/resolve/main/config.json'
    ),
    # See all TimeSeriesTransformer models at https://huggingface.co/models?filter=time_series_transformer
}


class _lowerCAmelCase						(    __UpperCAmelCase       ):
						SCREAMING_SNAKE_CASE_:					Any 					=				'time_series_transformer'
						SCREAMING_SNAKE_CASE_:					List[str] 					=				{
						    'hidden_size': 'd_model',
						    'num_attention_heads': 'encoder_attention_heads',
						    'num_hidden_layers': 'encoder_layers',
						}







						def __init__(    self   ,				lowerCAmelCase_ = None   ,				lowerCAmelCase_ = None   ,				lowerCAmelCase_ = "student_t"   ,				lowerCAmelCase_ = "nll"   ,				lowerCAmelCase_ = 1   ,				lowerCAmelCase_ = [1, 2, 3, 4, 5, 6, 7]   ,				lowerCAmelCase_ = "mean"   ,				lowerCAmelCase_ = 0   ,				lowerCAmelCase_ = 0   ,				lowerCAmelCase_ = 0   ,				lowerCAmelCase_ = 0   ,				lowerCAmelCase_ = None   ,				lowerCAmelCase_ = None   ,				lowerCAmelCase_ = 3_2   ,				lowerCAmelCase_ = 3_2   ,				lowerCAmelCase_ = 2   ,				lowerCAmelCase_ = 2   ,				lowerCAmelCase_ = 2   ,				lowerCAmelCase_ = 2   ,				lowerCAmelCase_ = True   ,				lowerCAmelCase_ = "gelu"   ,				lowerCAmelCase_ = 6_4   ,				lowerCAmelCase_ = 0.1   ,				lowerCAmelCase_ = 0.1   ,				lowerCAmelCase_ = 0.1   ,				lowerCAmelCase_ = 0.1   ,				lowerCAmelCase_ = 0.1   ,				lowerCAmelCase_ = 1_0_0   ,				lowerCAmelCase_ = 0.02   ,				lowerCAmelCase_=True   ,				**lowerCAmelCase_   ,				)		->							str:
										# time series specific configuration
										_SCREAMING_SNAKE_CASE   :		List[str]       		=    prediction_length
										_SCREAMING_SNAKE_CASE   :		Tuple       		=    context_length or prediction_length
										_SCREAMING_SNAKE_CASE   :		Any       		=    distribution_output
										_SCREAMING_SNAKE_CASE   :		List[str]       		=    loss
										_SCREAMING_SNAKE_CASE   :		Any       		=    input_size
										_SCREAMING_SNAKE_CASE   :		Any       		=    num_time_features
										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    lags_sequence
										_SCREAMING_SNAKE_CASE   :		Dict       		=    scaling
										_SCREAMING_SNAKE_CASE   :		Any       		=    num_dynamic_real_features
										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    num_static_real_features
										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    num_static_categorical_features
										if cardinality and num_static_categorical_features > 0:
														if len(lowerCAmelCase_							) != num_static_categorical_features:
																		raise ValueError(
																		    'The cardinality should be a list of the same length as `num_static_categorical_features`'							)
														_SCREAMING_SNAKE_CASE   :		List[str]       		=    cardinality
										else:
														_SCREAMING_SNAKE_CASE   :		Dict       		=    [0]
										if embedding_dimension and num_static_categorical_features > 0:
														if len(lowerCAmelCase_							) != num_static_categorical_features:
																		raise ValueError(
																		    'The embedding dimension should be a list of the same length as `num_static_categorical_features`'							)
														_SCREAMING_SNAKE_CASE   :		int       		=    embedding_dimension
										else:
														_SCREAMING_SNAKE_CASE   :		List[str]       		=    [min(5_0   ,				(cat + 1) // 2							) for cat in self.cardinality]
										_SCREAMING_SNAKE_CASE   :		str       		=    num_parallel_samples

										# Transformer architecture configuration
										_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    input_size * len(lowerCAmelCase_							) + self._number_of_features
										_SCREAMING_SNAKE_CASE   :		int       		=    d_model
										_SCREAMING_SNAKE_CASE   :		str       		=    encoder_attention_heads
										_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    decoder_attention_heads
										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    encoder_ffn_dim
										_SCREAMING_SNAKE_CASE   :		Dict       		=    decoder_ffn_dim
										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    encoder_layers
										_SCREAMING_SNAKE_CASE   :		Tuple       		=    decoder_layers

										_SCREAMING_SNAKE_CASE   :		Dict       		=    dropout
										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    attention_dropout
										_SCREAMING_SNAKE_CASE   :		Tuple       		=    activation_dropout
										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    encoder_layerdrop
										_SCREAMING_SNAKE_CASE   :		List[str]       		=    decoder_layerdrop

										_SCREAMING_SNAKE_CASE   :		Dict       		=    activation_function
										_SCREAMING_SNAKE_CASE   :		str       		=    init_std

										_SCREAMING_SNAKE_CASE   :		Tuple       		=    use_cache

										super().__init__(is_encoder_decoder=lowerCAmelCase_   ,				**lowerCAmelCase_							)







						@property
						def 			A      (    self							)		->							int:
										return (
										    sum(self.embedding_dimension							)
										    + self.num_dynamic_real_features
										    + self.num_time_features
										    + self.num_static_real_features
										    + self.input_size * 2  # the log1p(abs(loc)) and log(scale) features
										)

