
import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowercase__		:Tuple   =    logging.get_logger(__name__)

lowercase__		:Tuple   =    {"vocab_file": "spiece.model"}

lowercase__		:Dict   =    {
    "vocab_file": {
        "google/bigbird-roberta-base": "https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model",
        "google/bigbird-roberta-large": (
            "https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model"
        ),
        "google/bigbird-base-trivia-itc": (
            "https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model"
        ),
    }
}

lowercase__		:Optional[Any]   =    {
    "google/bigbird-roberta-base": 4096,
    "google/bigbird-roberta-large": 4096,
    "google/bigbird-base-trivia-itc": 4096,
}







class 				lowercase		(					SCREAMING_SNAKE_CASE__		):
							lowercase_       :       Dict       					=VOCAB_FILES_NAMES
							lowercase_       :       Dict       					=PRETRAINED_VOCAB_FILES_MAP
							lowercase_       :       Tuple       					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_       :       Union[str, Any]       					=['''input_ids''', '''attention_mask''']
							lowercase_       :       List[int]       					=[]




							def __init__(				self   ,A__   ,A__="<unk>"   ,A__="<s>"   ,A__="</s>"   ,A__="<pad>"   ,A__="[SEP]"   ,A__="[MASK]"   ,A__="[CLS]"   ,A__ = None   ,**A__   ,):
												lowercase							=						AddedToken(A__   ,lstrip=A__   ,rstrip=A__) if isinstance(A__   ,A__) else bos_token
												lowercase							=						AddedToken(A__   ,lstrip=A__   ,rstrip=A__) if isinstance(A__   ,A__) else eos_token
												lowercase							=						AddedToken(A__   ,lstrip=A__   ,rstrip=A__) if isinstance(A__   ,A__) else unk_token
												lowercase							=						AddedToken(A__   ,lstrip=A__   ,rstrip=A__) if isinstance(A__   ,A__) else pad_token
												lowercase							=						AddedToken(A__   ,lstrip=A__   ,rstrip=A__) if isinstance(A__   ,A__) else cls_token
												lowercase							=						AddedToken(A__   ,lstrip=A__   ,rstrip=A__) if isinstance(A__   ,A__) else sep_token

												# Mask token behave like a normal word, i.e. include the space before it
												lowercase							=						AddedToken(A__   ,lstrip=A__   ,rstrip=A__) if isinstance(A__   ,A__) else mask_token

												lowercase							=						{} if sp_model_kwargs is None else sp_model_kwargs

												super().__init__(
												    bos_token=A__   ,eos_token=A__   ,unk_token=A__   ,pad_token=A__   ,sep_token=A__   ,mask_token=A__   ,cls_token=A__   ,sp_model_kwargs=self.sp_model_kwargs   ,**A__   ,)

												lowercase							=						vocab_file

												lowercase							=						spm.SentencePieceProcessor(**self.sp_model_kwargs)
												self.sp_model.Load(A__)




							@property
							def  A__      (				self):
												return self.sp_model.get_piece_size()




							def  A__      (				self):
												lowercase							=						{self.convert_ids_to_tokens(A__): i for i in range(self.vocab_size)}
												vocab.update(self.added_tokens_encoder)
												return vocab




							def __getstate__(				self):
												lowercase							=						self.__dict__.copy()
												lowercase							=						None
												return state




							def __setstate__(				self   ,A__):
												lowercase							=						d

												# for backward compatibility
												if not hasattr(self   ,'''sp_model_kwargs'''):
																	lowercase							=						{}

												lowercase							=						spm.SentencePieceProcessor(**self.sp_model_kwargs)
												self.sp_model.Load(self.vocab_file)




							def  A__      (				self   ,A__):
												return self.sp_model.encode(A__   ,out_type=A__)




							def  A__      (				self   ,A__):
												return self.sp_model.piece_to_id(A__)




							def  A__      (				self   ,A__):
												lowercase							=						self.sp_model.IdToPiece(A__)
												return token




							def  A__      (				self   ,A__):
												lowercase							=						[]
												lowercase							=						''''''
												lowercase							=						False
												for token in tokens:
																	# make sure that special tokens are not decoded using sentencepiece model
																	if token in self.all_special_tokens:
																						if not prev_is_special:
																											out_string += " "
																						out_string += self.sp_model.decode(A__) + token
																						lowercase							=						True
																						lowercase							=						[]
																	else:
																						current_sub_tokens.append(A__)
																						lowercase							=						False
												out_string += self.sp_model.decode(A__)
												return out_string.strip()




							def  A__      (				self   ,A__   ,A__ = False   ,A__ = None   ,A__ = True   ,**A__   ,):
												lowercase							=						kwargs.pop('''use_source_tokenizer'''   ,A__)

												lowercase							=						self.convert_ids_to_tokens(A__   ,skip_special_tokens=A__)

												# To avoid mixing byte-level and unicode for byte-level BPT
												# we need to build string separately for added tokens and byte-level tokens
												# cf. https://github.com/huggingface/transformers/issues/1133
												lowercase							=						[]
												lowercase							=						[]
												for token in filtered_tokens:
																	if skip_special_tokens and token in self.all_special_ids:
																						continue
																	if token in self.added_tokens_encoder:
																						if current_sub_text:
																											sub_texts.append(self.convert_tokens_to_string(A__))
																											lowercase							=						[]
																						sub_texts.append(A__)
																	else:
																						current_sub_text.append(A__)
												if current_sub_text:
																	sub_texts.append(self.convert_tokens_to_string(A__))

												# Mimic the behavior of the Rust tokenizer:
												# No space before [MASK] and [SEP]
												if spaces_between_special_tokens:
																	lowercase							=						re.sub(r''' (\[(MASK|SEP)\])'''   ,r'''\1'''   ,''' '''.join(A__))
												else:
																	lowercase							=						''''''.join(A__)

												lowercase							=						(
												    clean_up_tokenization_spaces
												    if clean_up_tokenization_spaces is not None
												    else self.clean_up_tokenization_spaces
												)
												if clean_up_tokenization_spaces:
																	lowercase							=						self.clean_up_tokenization(A__)
																	return clean_text
												else:
																	return text




							def  A__      (				self   ,A__   ,A__ = None):
												if not os.path.isdir(A__):
																	logger.error(f'Vocabulary path ({save_directory}) should be a directory')
																	return
												lowercase							=						os.path.join(
												    A__   ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

												if os.path.abspath(self.vocab_file) != os.path.abspath(A__) and os.path.isfile(self.vocab_file):
																	copyfile(self.vocab_file   ,A__)
												elif not os.path.isfile(self.vocab_file):
																	with open(A__   ,'''wb''') as fi:
																						lowercase							=						self.sp_model.serialized_model_proto()
																						fi.write(A__)

												return (out_vocab_file,)




							def  A__      (				self   ,A__   ,A__ = None):
												if token_ids_a is None:
																	return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
												lowercase							=						[self.cls_token_id]
												lowercase							=						[self.sep_token_id]
												return cls + token_ids_a + sep + token_ids_a + sep




							def  A__      (				self   ,A__   ,A__ = None   ,A__ = False):
												if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=A__   ,token_ids_a=A__   ,already_has_special_tokens=A__)

												if token_ids_a is None:
																	return [1] + ([0] * len(A__)) + [1]
												return [1] + ([0] * len(A__)) + [1] + ([0] * len(A__)) + [1]





							def  A__      (				self   ,A__   ,A__ = None):
												lowercase							=						[self.sep_token_id]
												lowercase							=						[self.cls_token_id]
												if token_ids_a is None:
																	return len(cls + token_ids_a + sep) * [0]
												return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


lowercase__		:List[Any]   =    logging.get_logger(__name__)

lowercase__		:Optional[Any]   =    {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

lowercase__		:Union[str, Any]   =    {
    "vocab_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"
        ),
        "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"
        ),
        "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt",
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"
        ),
        "bert-base-multilingual-cased": (
            "https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-cased": (
            "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"
        ),
    },
}

lowercase__		:str   =    {
    "bert-base-uncased": 512,
    "bert-large-uncased": 512,
    "bert-base-cased": 512,
    "bert-large-cased": 512,
    "bert-base-multilingual-uncased": 512,
    "bert-base-multilingual-cased": 512,
    "bert-base-chinese": 512,
    "bert-base-german-cased": 512,
    "bert-large-uncased-whole-word-masking": 512,
    "bert-large-cased-whole-word-masking": 512,
    "bert-large-uncased-whole-word-masking-finetuned-squad": 512,
    "bert-large-cased-whole-word-masking-finetuned-squad": 512,
    "bert-base-cased-finetuned-mrpc": 512,
    "bert-base-german-dbmdz-cased": 512,
    "bert-base-german-dbmdz-uncased": 512,
    "TurkuNLP/bert-base-finnish-cased-v1": 512,
    "TurkuNLP/bert-base-finnish-uncased-v1": 512,
    "wietsedv/bert-base-dutch-cased": 512,
}

lowercase__		:Dict   =    {
    "bert-base-uncased": {"do_lower_case": True},
    "bert-large-uncased": {"do_lower_case": True},
    "bert-base-cased": {"do_lower_case": False},
    "bert-large-cased": {"do_lower_case": False},
    "bert-base-multilingual-uncased": {"do_lower_case": True},
    "bert-base-multilingual-cased": {"do_lower_case": False},
    "bert-base-chinese": {"do_lower_case": False},
    "bert-base-german-cased": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False},
    "bert-base-cased-finetuned-mrpc": {"do_lower_case": False},
    "bert-base-german-dbmdz-cased": {"do_lower_case": False},
    "bert-base-german-dbmdz-uncased": {"do_lower_case": True},
    "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False},
    "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True},
    "wietsedv/bert-base-dutch-cased": {"do_lower_case": False},
}







class 				lowercase		(					SCREAMING_SNAKE_CASE__		):
							lowercase_       :       str       					=VOCAB_FILES_NAMES
							lowercase_       :       Dict       					=PRETRAINED_VOCAB_FILES_MAP
							lowercase_       :       Optional[int]       					=PRETRAINED_INIT_CONFIGURATION
							lowercase_       :       Tuple       					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_       :       Any       					=BertTokenizer




							def __init__(				self   ,A__=None   ,A__=None   ,A__=True   ,A__="[UNK]"   ,A__="[SEP]"   ,A__="[PAD]"   ,A__="[CLS]"   ,A__="[MASK]"   ,A__=True   ,A__=None   ,**A__   ,):
												super().__init__(
												    A__   ,tokenizer_file=A__   ,do_lower_case=A__   ,unk_token=A__   ,sep_token=A__   ,pad_token=A__   ,cls_token=A__   ,mask_token=A__   ,tokenize_chinese_chars=A__   ,strip_accents=A__   ,**A__   ,)

												lowercase							=						json.loads(self.backend_tokenizer.normalizer.__getstate__())
												if (
												    normalizer_state.get('''lowercase'''   ,A__) != do_lower_case
												    or normalizer_state.get('''strip_accents'''   ,A__) != strip_accents
												    or normalizer_state.get('''handle_chinese_chars'''   ,A__) != tokenize_chinese_chars
												):
																	lowercase							=						getattr(A__   ,normalizer_state.pop('''type'''))
																	lowercase							=						do_lower_case
																	lowercase							=						strip_accents
																	lowercase							=						tokenize_chinese_chars
																	lowercase							=						normalizer_class(**A__)

												lowercase							=						do_lower_case




							def  A__      (				self   ,A__   ,A__=None):
												lowercase							=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

												if token_ids_a:
																	output += token_ids_a + [self.sep_token_id]

												return output




							def  A__      (				self   ,A__   ,A__ = None):
												lowercase							=						[self.sep_token_id]
												lowercase							=						[self.cls_token_id]
												if token_ids_a is None:
																	return len(cls + token_ids_a + sep) * [0]
												return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]





							def  A__      (				self   ,A__   ,A__ = None):
												lowercase							=						self._tokenizer.model.save(A__   ,name=A__)
												return tuple(A__)




