import torch
from torch import nn

class        UpperCamelCase						(				nn.Module				):





						def __init__(self			,     __UpperCamelCase			,     __UpperCamelCase			,     __UpperCamelCase			,     __UpperCamelCase			,     __UpperCamelCase=1			,     __UpperCamelCase=False						)				->				Optional[int]:
												super().__init__()

												UpperCamelCase_		:							List[str]  =		n_token
												UpperCamelCase_		:							List[Any]  =		d_embed
												UpperCamelCase_		:							List[Any]  =		d_proj

												UpperCamelCase_		:							Optional[Any]  =		cutoffs + [n_token]
												UpperCamelCase_		:							Tuple  =		[0] + self.cutoffs
												UpperCamelCase_		:							Union[str, Any]  =		div_val

												UpperCamelCase_		:							str  =		self.cutoffs[0]
												UpperCamelCase_		:							Tuple  =		len(self.cutoffs						) - 1
												UpperCamelCase_		:							int  =		self.shortlist_size + self.n_clusters

												if self.n_clusters > 0:
																		UpperCamelCase_		:							Union[str, Any]  =		nn.Parameter(torch.zeros(self.n_clusters			,     self.d_embed						)						)
																		UpperCamelCase_		:							Dict  =		nn.Parameter(torch.zeros(self.n_clusters						)						)

												UpperCamelCase_		:							Tuple  =		nn.ModuleList()
												UpperCamelCase_		:							List[Any]  =		nn.ParameterList()

												if div_val == 1:
																		for i in range(len(self.cutoffs						)						):
																								if d_proj != d_embed:
																														self.out_projs.append(nn.Parameter(torch.FloatTensor(__UpperCamelCase			,     __UpperCamelCase						)						)						)
																								else:
																														self.out_projs.append(__UpperCamelCase						)

																		self.out_layers.append(nn.Linear(__UpperCamelCase			,     __UpperCamelCase						)						)
												else:
																		for i in range(len(self.cutoffs						)						):
																								UpperCamelCase_,UpperCamelCase_		:							Optional[Any]  =		self.cutoff_ends[i], self.cutoff_ends[i + 1]
																								UpperCamelCase_		:							Dict  =		d_embed // (div_val**i)

																								self.out_projs.append(nn.Parameter(torch.FloatTensor(__UpperCamelCase			,     __UpperCamelCase						)						)						)

																								self.out_layers.append(nn.Linear(__UpperCamelCase			,     r_idx - l_idx						)						)

												UpperCamelCase_		:							Any  =		keep_order





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase			,     __UpperCamelCase			,     __UpperCamelCase						)				->				Optional[int]:
												if proj is None:
																		UpperCamelCase_		:							Optional[int]  =		nn.functional.linear(__UpperCamelCase			,     __UpperCamelCase			,     bias=__UpperCamelCase						)
												else:
																		# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
																		UpperCamelCase_		:							Optional[Any]  =		nn.functional.linear(__UpperCamelCase			,     proj.t().contiguous()						)
																		UpperCamelCase_		:							Optional[int]  =		nn.functional.linear(__UpperCamelCase			,     __UpperCamelCase			,     bias=__UpperCamelCase						)
																		# else:
																		#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
																		#     if bias is not None:
																		#         logit = logit + bias

												return logit





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase=None			,     __UpperCamelCase=False						)				->				List[Any]:

												if labels is not None:
																		# Shift so that tokens < n predict n
																		UpperCamelCase_		:							str  =		hidden[..., :-1, :].contiguous()
																		UpperCamelCase_		:							List[Any]  =		labels[..., 1:].contiguous()
																		UpperCamelCase_		:							Dict  =		hidden.view(-1			,     hidden.size(-1						)						)
																		UpperCamelCase_		:							Union[str, Any]  =		labels.view(-1						)
																		if hidden.size(0						) != labels.size(0						):
																								raise RuntimeError("""Input and labels should have the same size in the batch dimension."""						)
												else:
																		UpperCamelCase_		:							Optional[int]  =		hidden.view(-1			,     hidden.size(-1						)						)

												if self.n_clusters == 0:
																		UpperCamelCase_		:							Union[str, Any]  =		self._compute_logit(__UpperCamelCase			,     self.out_layers[0].weight			,     self.out_layers[0].bias			,     self.out_projs[0]						)
																		if labels is not None:
																								UpperCamelCase_		:							int  =		labels != -100
																								UpperCamelCase_		:							Optional[int]  =		torch.zeros_like(__UpperCamelCase			,     dtype=hidden.dtype			,     device=hidden.device						)
																								UpperCamelCase_		:							int  =		(
																								    -nn.functional.log_softmax(__UpperCamelCase			,     dim=-1						)[mask].gather(1			,     labels[mask].unsqueeze(1						)						).squeeze(1						)
																								)
																		else:
																								UpperCamelCase_		:							Optional[Any]  =		nn.functional.log_softmax(__UpperCamelCase			,     dim=-1						)
												else:
																		# construct weights and biases
																		UpperCamelCase_,UpperCamelCase_		:							Union[str, Any]  =		[], []
																		for i in range(len(self.cutoffs						)						):
																								if self.div_val == 1:
																														UpperCamelCase_,UpperCamelCase_		:							List[str]  =		self.cutoff_ends[i], self.cutoff_ends[i + 1]
																														UpperCamelCase_		:							List[Any]  =		self.out_layers[0].weight[l_idx:r_idx]
																														UpperCamelCase_		:							List[str]  =		self.out_layers[0].bias[l_idx:r_idx]
																								else:
																														UpperCamelCase_		:							int  =		self.out_layers[i].weight
																														UpperCamelCase_		:							List[Any]  =		self.out_layers[i].bias

																								if i == 0:
																														UpperCamelCase_		:							List[str]  =		torch.cat([weight_i, self.cluster_weight]			,     dim=0						)
																														UpperCamelCase_		:							int  =		torch.cat([bias_i, self.cluster_bias]			,     dim=0						)

																								weights.append(__UpperCamelCase						)
																								biases.append(__UpperCamelCase						)

																		UpperCamelCase_,UpperCamelCase_,UpperCamelCase_		:							List[str]  =		weights[0], biases[0], self.out_projs[0]

																		UpperCamelCase_		:							Dict  =		self._compute_logit(__UpperCamelCase			,     __UpperCamelCase			,     __UpperCamelCase			,     __UpperCamelCase						)
																		UpperCamelCase_		:							int  =		nn.functional.log_softmax(__UpperCamelCase			,     dim=1						)

																		if labels is None:
																								UpperCamelCase_		:							List[str]  =		hidden.new_empty((head_logit.size(0						), self.n_token)						)
																		else:
																								UpperCamelCase_		:							Dict  =		torch.zeros_like(__UpperCamelCase			,     dtype=hidden.dtype			,     device=hidden.device						)

																		UpperCamelCase_		:							Any  =		0
																		UpperCamelCase_		:							int  =		[0] + self.cutoffs
																		for i in range(len(__UpperCamelCase						) - 1						):
																								UpperCamelCase_,UpperCamelCase_		:							str  =		cutoff_values[i], cutoff_values[i + 1]

																								if labels is not None:
																														UpperCamelCase_		:							Optional[int]  =		(labels >= l_idx) & (labels < r_idx)
																														UpperCamelCase_		:							Dict  =		mask_i.nonzero().squeeze()

																														if indices_i.numel() == 0:
																																				continue

																														UpperCamelCase_		:							Optional[Any]  =		labels.index_select(0			,     __UpperCamelCase						) - l_idx
																														UpperCamelCase_		:							Optional[Any]  =		head_logprob.index_select(0			,     __UpperCamelCase						)
																														UpperCamelCase_		:							Dict  =		hidden.index_select(0			,     __UpperCamelCase						)
																								else:
																														UpperCamelCase_		:							Union[str, Any]  =		hidden

																								if i == 0:
																														if labels is not None:
																																				UpperCamelCase_		:							Union[str, Any]  =		head_logprob_i.gather(1			,     target_i[:, None]						).squeeze(1						)
																														else:
																																				UpperCamelCase_		:							Optional[Any]  =		head_logprob[:, : self.cutoffs[0]]
																								else:
																														UpperCamelCase_,UpperCamelCase_,UpperCamelCase_		:							List[Any]  =		weights[i], biases[i], self.out_projs[i]

																														UpperCamelCase_		:							Optional[Any]  =		self._compute_logit(__UpperCamelCase			,     __UpperCamelCase			,     __UpperCamelCase			,     __UpperCamelCase						)
																														UpperCamelCase_		:							Tuple  =		nn.functional.log_softmax(__UpperCamelCase			,     dim=1						)
																														UpperCamelCase_		:							Optional[Any]  =		self.cutoffs[0] + i - 1  # No probability for the head cluster
																														if labels is not None:
																																				UpperCamelCase_		:							Any  =		head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
																																				    1			,     target_i[:, None]						).squeeze(1						)
																														else:
																																				UpperCamelCase_		:							Any  =		head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
																																				UpperCamelCase_		:							Tuple  =		logprob_i

																								if labels is not None:
																														if (hasattr(self			,     """keep_order"""						) and self.keep_order) or keep_order:
																																				out.index_copy_(0			,     __UpperCamelCase			,     -logprob_i						)
																														else:
																																				out[offset : offset + logprob_i.size(0						)].copy_(-logprob_i						)
																														offset += logprob_i.size(0						)

												return out





						def        A_							(self			,     __UpperCamelCase						)				->				Tuple:
												if self.n_clusters == 0:
																		UpperCamelCase_		:							Optional[int]  =		self._compute_logit(__UpperCamelCase			,     self.out_layers[0].weight			,     self.out_layers[0].bias			,     self.out_projs[0]						)
																		return nn.functional.log_softmax(__UpperCamelCase			,     dim=-1						)
												else:
																		# construct weights and biases
																		UpperCamelCase_,UpperCamelCase_		:							List[str]  =		[], []
																		for i in range(len(self.cutoffs						)						):
																								if self.div_val == 1:
																														UpperCamelCase_,UpperCamelCase_		:							str  =		self.cutoff_ends[i], self.cutoff_ends[i + 1]
																														UpperCamelCase_		:							List[str]  =		self.out_layers[0].weight[l_idx:r_idx]
																														UpperCamelCase_		:							Tuple  =		self.out_layers[0].bias[l_idx:r_idx]
																								else:
																														UpperCamelCase_		:							Optional[int]  =		self.out_layers[i].weight
																														UpperCamelCase_		:							Union[str, Any]  =		self.out_layers[i].bias

																								if i == 0:
																														UpperCamelCase_		:							List[Any]  =		torch.cat([weight_i, self.cluster_weight]			,     dim=0						)
																														UpperCamelCase_		:							Optional[Any]  =		torch.cat([bias_i, self.cluster_bias]			,     dim=0						)

																								weights.append(__UpperCamelCase						)
																								biases.append(__UpperCamelCase						)

																		UpperCamelCase_,UpperCamelCase_,UpperCamelCase_		:							List[str]  =		weights[0], biases[0], self.out_projs[0]
																		UpperCamelCase_		:							List[str]  =		self._compute_logit(__UpperCamelCase			,     __UpperCamelCase			,     __UpperCamelCase			,     __UpperCamelCase						)

																		UpperCamelCase_		:							Optional[int]  =		hidden.new_empty((head_logit.size(0						), self.n_token)						)
																		UpperCamelCase_		:							int  =		nn.functional.log_softmax(__UpperCamelCase			,     dim=1						)

																		UpperCamelCase_		:							Optional[int]  =		[0] + self.cutoffs
																		for i in range(len(__UpperCamelCase						) - 1						):
																								UpperCamelCase_,UpperCamelCase_		:							str  =		cutoff_values[i], cutoff_values[i + 1]

																								if i == 0:
																														UpperCamelCase_		:							Optional[int]  =		head_logprob[:, : self.cutoffs[0]]
																								else:
																														UpperCamelCase_,UpperCamelCase_,UpperCamelCase_		:							Tuple  =		weights[i], biases[i], self.out_projs[i]

																														UpperCamelCase_		:							Any  =		self._compute_logit(__UpperCamelCase			,     __UpperCamelCase			,     __UpperCamelCase			,     __UpperCamelCase						)
																														UpperCamelCase_		:							int  =		nn.functional.log_softmax(__UpperCamelCase			,     dim=1						)

																														UpperCamelCase_		:							Union[str, Any]  =		head_logprob[:, -i] + tail_logprob_i
																														UpperCamelCase_		:							Union[str, Any]  =		logprob_i

																		return out







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


SCREAMING_SNAKE_CASE		:     Union[str, Any]         =	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE		:     Dict         =	{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

SCREAMING_SNAKE_CASE		:     Dict         =	[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

SCREAMING_SNAKE_CASE		:     int         =	{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
SCREAMING_SNAKE_CASE		:     Any         =	{F'''funnel-transformer/{name}''': 512 for name in _model_names}
SCREAMING_SNAKE_CASE		:     Tuple         =	{F'''funnel-transformer/{name}''': {"do_lower_case": True} for name in _model_names}

class        UpperCamelCase						(				__a				):
						a__		:Dict      =			VOCAB_FILES_NAMES
						a__		:Dict      =			PRETRAINED_VOCAB_FILES_MAP
						a__		:Dict      =			PRETRAINED_INIT_CONFIGURATION
						a__		:str      =			FunnelTokenizer
						a__		:List[str]      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						a__		:int      =			2





						def __init__(self			,     __UpperCamelCase=None			,     __UpperCamelCase=None			,     __UpperCamelCase=True			,     __UpperCamelCase="<unk>"			,     __UpperCamelCase="<sep>"			,     __UpperCamelCase="<pad>"			,     __UpperCamelCase="<cls>"			,     __UpperCamelCase="<mask>"			,     __UpperCamelCase="<s>"			,     __UpperCamelCase="</s>"			,     __UpperCamelCase=True			,     __UpperCamelCase=True			,     __UpperCamelCase=None			,     __UpperCamelCase="##"			,     **__UpperCamelCase			,     )				->				List[Any]:
												super().__init__(
												    __UpperCamelCase			,     tokenizer_file=__UpperCamelCase			,     do_lower_case=__UpperCamelCase			,     unk_token=__UpperCamelCase			,     sep_token=__UpperCamelCase			,     pad_token=__UpperCamelCase			,     cls_token=__UpperCamelCase			,     mask_token=__UpperCamelCase			,     bos_token=__UpperCamelCase			,     eos_token=__UpperCamelCase			,     clean_text=__UpperCamelCase			,     tokenize_chinese_chars=__UpperCamelCase			,     strip_accents=__UpperCamelCase			,     wordpieces_prefix=__UpperCamelCase			,     **__UpperCamelCase			,     )

												UpperCamelCase_		:							List[str]  =		json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
												if (
												    normalizer_state.get("""lowercase"""			,     __UpperCamelCase						) != do_lower_case
												    or normalizer_state.get("""strip_accents"""			,     __UpperCamelCase						) != strip_accents
												    or normalizer_state.get("""handle_chinese_chars"""			,     __UpperCamelCase						) != tokenize_chinese_chars
												):
																		UpperCamelCase_		:							List[Any]  =		getattr(__UpperCamelCase			,     normalizer_state.pop("""type"""						)						)
																		UpperCamelCase_		:							Union[str, Any]  =		do_lower_case
																		UpperCamelCase_		:							Tuple  =		strip_accents
																		UpperCamelCase_		:							Dict  =		tokenize_chinese_chars
																		UpperCamelCase_		:							Union[str, Any]  =		normalizer_class(**__UpperCamelCase						)

												UpperCamelCase_		:							Dict  =		do_lower_case





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase=None						)				->				Dict:
												UpperCamelCase_		:							Any  =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

												if token_ids_a:
																		output += token_ids_a + [self.sep_token_id]

												return output





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase = None						)				->				List[int]:
												UpperCamelCase_		:							Optional[Any]  =		[self.sep_token_id]
												UpperCamelCase_		:							Any  =		[self.cls_token_id]
												if token_ids_a is None:
																		return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0]
												return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase = None						)				->				Tuple[str]:
												UpperCamelCase_		:							int  =		self._tokenizer.model.save(__UpperCamelCase			,     name=__UpperCamelCase						)
												return tuple(__UpperCamelCase						)






