




import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__A        =    logging.get_logger(__name__)
__A        =    {"vocab_file": "spiece.model"}

__A        =    {
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

__A        =    {
    "albert-base-v1": 5_12,
    "albert-large-v1": 5_12,
    "albert-xlarge-v1": 5_12,
    "albert-xxlarge-v1": 5_12,
    "albert-base-v2": 5_12,
    "albert-large-v2": 5_12,
    "albert-xlarge-v2": 5_12,
    "albert-xxlarge-v2": 5_12,
}

__A        =    "▁"







class     __lowerCAmelCase (       __magic_name__ ):

			"""simple docstring"""







			snake_case_   		=       VOCAB_FILES_NAMES
			snake_case_   		=       PRETRAINED_VOCAB_FILES_MAP
			snake_case_   		=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES


			def __init__(      self			,						lowerCamelCase__			,						lowerCamelCase__=True			,						lowerCamelCase__=True			,						lowerCamelCase__=False			,						lowerCamelCase__="[CLS]"			,						lowerCamelCase__="[SEP]"			,						lowerCamelCase__="<unk>"			,						lowerCamelCase__="[SEP]"			,						lowerCamelCase__="<pad>"			,						lowerCamelCase__="[CLS]"			,						lowerCamelCase__="[MASK]"			,						lowerCamelCase__ = None			,						**lowerCamelCase__			,						)    ->      None:



							'''simple docstring'''
							# Mask token behave like a normal word, i.e. include the space before it and
							# is included in the raw text, there should be a match in a non-normalized sentence.
							__lowerCamelCase		       =    (
							    AddedToken(lowerCamelCase__			,						lstrip=lowerCamelCase__			,						rstrip=lowerCamelCase__			,						normalized=lowerCamelCase__							)
							    if isinstance(lowerCamelCase__			,						lowerCamelCase__							)
							    else mask_token
							)

							__lowerCamelCase		       =    {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    do_lower_case=lowerCamelCase__			,						remove_space=lowerCamelCase__			,						keep_accents=lowerCamelCase__			,						bos_token=lowerCamelCase__			,						eos_token=lowerCamelCase__			,						unk_token=lowerCamelCase__			,						sep_token=lowerCamelCase__			,						pad_token=lowerCamelCase__			,						cls_token=lowerCamelCase__			,						mask_token=lowerCamelCase__			,						sp_model_kwargs=self.sp_model_kwargs			,						**lowerCamelCase__			,						)

							__lowerCamelCase		       =    do_lower_case
							__lowerCamelCase		       =    remove_space
							__lowerCamelCase		       =    keep_accents
							__lowerCamelCase		       =    vocab_file

							__lowerCamelCase		       =    spm.SentencePieceProcessor(**self.sp_model_kwargs							)
							self.sp_model.Load(lowerCamelCase__							)


			@property
			def       lowercase_    (      self							)    ->      List[str]:



							'''simple docstring'''
							return len(self.sp_model							)


			def       lowercase_    (      self							)    ->      Dict:



							'''simple docstring'''
							__lowerCamelCase		       =    {self.convert_ids_to_tokens(lowerCamelCase__							): i for i in range(self.vocab_size							)}
							vocab.update(self.added_tokens_encoder							)
							return vocab


			def __getstate__(      self							)    ->      int:



							'''simple docstring'''
							__lowerCamelCase		       =    self.__dict__.copy()
							__lowerCamelCase		       =    None
							return state


			def __setstate__(      self			,						lowerCamelCase__							)    ->      List[Any]:



							'''simple docstring'''
							__lowerCamelCase		       =    d

							# for backward compatibility
							if not hasattr(self			,						'sp_model_kwargs'							):
											__lowerCamelCase		       =    {}

							__lowerCamelCase		       =    spm.SentencePieceProcessor(**self.sp_model_kwargs							)
							self.sp_model.Load(self.vocab_file							)


			def       lowercase_    (      self			,						lowerCamelCase__							)    ->      Optional[Any]:



							'''simple docstring'''
							if self.remove_space:
											__lowerCamelCase		       =    ' '.join(inputs.strip().split()							)
							else:
											__lowerCamelCase		       =    inputs
							__lowerCamelCase		       =    outputs.replace('``'			,						'"'							).replace('\'\''			,						'"'							)

							if not self.keep_accents:
											__lowerCamelCase		       =    unicodedata.normalize('NFKD'			,						lowerCamelCase__							)
											__lowerCamelCase		       =    ''.join([c for c in outputs if not unicodedata.combining(lowerCamelCase__							)]							)
							if self.do_lower_case:
											__lowerCamelCase		       =    outputs.lower()

							return outputs


			def       lowercase_    (      self			,						lowerCamelCase__							)    ->      List[str]:



							'''simple docstring'''
							__lowerCamelCase		       =    self.preprocess_text(lowerCamelCase__							)
							__lowerCamelCase		       =    self.sp_model.encode(lowerCamelCase__			,						out_type=lowerCamelCase__							)
							__lowerCamelCase		       =    []
							for piece in pieces:
											if len(lowerCamelCase__							) > 1 and piece[-1] == str(','							) and piece[-2].isdigit():
															__lowerCamelCase		       =    self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCamelCase__			,						''							)							)
															if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																			if len(cur_pieces[0]							) == 1:
																							__lowerCamelCase		       =    cur_pieces[1:]
																			else:
																							__lowerCamelCase		       =    cur_pieces[0][1:]
															cur_pieces.append(piece[-1]							)
															new_pieces.extend(lowerCamelCase__							)
											else:
															new_pieces.append(lowerCamelCase__							)

							return new_pieces


			def       lowercase_    (      self			,						lowerCamelCase__							)    ->      str:



							'''simple docstring'''
							return self.sp_model.PieceToId(lowerCamelCase__							)


			def       lowercase_    (      self			,						lowerCamelCase__							)    ->      Optional[int]:



							'''simple docstring'''
							return self.sp_model.IdToPiece(lowerCamelCase__							)


			def       lowercase_    (      self			,						lowerCamelCase__							)    ->      Optional[Any]:



							'''simple docstring'''
							__lowerCamelCase		       =    []
							__lowerCamelCase		       =    ''
							__lowerCamelCase		       =    False
							for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
															if not prev_is_special:
																			out_string += " "
															out_string += self.sp_model.decode(lowerCamelCase__							) + token
															__lowerCamelCase		       =    True
															__lowerCamelCase		       =    []
											else:
															current_sub_tokens.append(lowerCamelCase__							)
															__lowerCamelCase		       =    False
							out_string += self.sp_model.decode(lowerCamelCase__							)
							return out_string.strip()


			def       lowercase_    (      self			,						lowerCamelCase__			,						lowerCamelCase__ = None							)    ->      List[int]:



							'''simple docstring'''
							__lowerCamelCase		       =    [self.sep_token_id]
							__lowerCamelCase		       =    [self.cls_token_id]
							if token_ids_a is None:
											return cls + token_ids_a + sep
							return cls + token_ids_a + sep + token_ids_a + sep


			def       lowercase_    (      self			,						lowerCamelCase__			,						lowerCamelCase__ = None			,						lowerCamelCase__ = False							)    ->      List[int]:



							'''simple docstring'''

							if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=lowerCamelCase__			,						token_ids_a=lowerCamelCase__			,						already_has_special_tokens=lowerCamelCase__							)

							if token_ids_a is not None:
											return [1] + ([0] * len(lowerCamelCase__							)) + [1] + ([0] * len(lowerCamelCase__							)) + [1]
							return [1] + ([0] * len(lowerCamelCase__							)) + [1]


			def       lowercase_    (      self			,						lowerCamelCase__			,						lowerCamelCase__ = None							)    ->      List[int]:



							'''simple docstring'''
							__lowerCamelCase		       =    [self.sep_token_id]
							__lowerCamelCase		       =    [self.cls_token_id]

							if token_ids_a is None:
											return len(cls + token_ids_a + sep							) * [0]
							return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]






			def       lowercase_    (      self			,						lowerCamelCase__			,						lowerCamelCase__ = None							)    ->      Tuple[str]:



							'''simple docstring'''
							if not os.path.isdir(lowerCamelCase__							):
											logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
											return
							__lowerCamelCase		       =    os.path.join(
							    lowerCamelCase__			,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)

							if os.path.abspath(self.vocab_file							) != os.path.abspath(lowerCamelCase__							) and os.path.isfile(self.vocab_file							):
											copyfile(self.vocab_file			,						lowerCamelCase__							)
							elif not os.path.isfile(self.vocab_file							):
											with open(lowerCamelCase__			,						'wb'							) as fi:
															__lowerCamelCase		       =    self.sp_model.serialized_model_proto()
															fi.write(lowerCamelCase__							)

							return (out_vocab_file,)








import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
__A        =    data_utils.TransfoXLTokenizer
__A        =    data_utils.TransfoXLCorpus
__A        =    data_utils
__A        =    data_utils







def 			lowerCamelCase_		(       UpperCamelCase__				:      List[str] ,     UpperCamelCase__				:      Optional[int] ,     UpperCamelCase__				:      Union[str, Any] ,     UpperCamelCase__				:      Optional[Any]      )     ->       Union[str, Any]:





				"""simple docstring"""


				if transfo_xl_dataset_file:
								# Convert a pre-processed corpus (see original TensorFlow repo)
								with open(UpperCamelCase__ ,     'rb'      ) as fp:
												__lowerCamelCase		       =    pickle.load(UpperCamelCase__ ,     encoding='latin1'      )
								# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
								__lowerCamelCase		       =    pytorch_dump_folder_path + '/' + VOCAB_FILES_NAMES['pretrained_vocab_file']
								print(F"""Save vocabulary to {pytorch_vocab_dump_path}"""      )
								__lowerCamelCase		       =    corpus.vocab.__dict__
								torch.save(UpperCamelCase__ ,     UpperCamelCase__      )

								__lowerCamelCase		       =    corpus.__dict__
								corpus_dict_no_vocab.pop('vocab' ,     UpperCamelCase__      )
								__lowerCamelCase		       =    pytorch_dump_folder_path + '/' + CORPUS_NAME
								print(F"""Save dataset to {pytorch_dataset_dump_path}"""      )
								torch.save(UpperCamelCase__ ,     UpperCamelCase__      )

				if tf_checkpoint_path:
								# Convert a pre-trained TensorFlow model
								__lowerCamelCase		       =    os.path.abspath(UpperCamelCase__      )
								__lowerCamelCase		       =    os.path.abspath(UpperCamelCase__      )

								print(F"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."""      )
								# Initialise PyTorch model
								if transfo_xl_config_file == "":
												__lowerCamelCase		       =    TransfoXLConfig()
								else:
												__lowerCamelCase		       =    TransfoXLConfig.from_json_file(UpperCamelCase__      )
								print(F"""Building PyTorch model from configuration: {config}"""      )
								__lowerCamelCase		       =    TransfoXLLMHeadModel(UpperCamelCase__      )

								__lowerCamelCase		       =    load_tf_weights_in_transfo_xl(UpperCamelCase__ ,     UpperCamelCase__ ,     UpperCamelCase__      )
								# Save pytorch-model
								__lowerCamelCase		       =    os.path.join(UpperCamelCase__ ,     UpperCamelCase__      )
								__lowerCamelCase		       =    os.path.join(UpperCamelCase__ ,     UpperCamelCase__      )
								print(F"""Save PyTorch model to {os.path.abspath(UpperCamelCase__      )}"""      )
								torch.save(model.state_dict() ,     UpperCamelCase__      )
								print(F"""Save configuration file to {os.path.abspath(UpperCamelCase__      )}"""      )
								with open(UpperCamelCase__ ,     'w' ,     encoding='utf-8'      ) as f:
												f.write(config.to_json_string()      )


if __name__ == "__main__":
					__A        =    argparse.ArgumentParser()
					parser.add_argument(
					    "--pytorch_dump_folder_path",
					    default=None,
					    type=str,
					    required=True,
					    help="Path to the folder to store the PyTorch model or dataset/vocab.",
					)
					parser.add_argument(
					    "--tf_checkpoint_path",
					    default="",
					    type=str,
					    help="An optional path to a TensorFlow checkpoint path to be converted.",
					)
					parser.add_argument(
					    "--transfo_xl_config_file",
					    default="",
					    type=str,
					    help=(
					        "An optional config json file corresponding to the pre-trained BERT model. \n"
					        "This specifies the model architecture."
					    ),
					)
					parser.add_argument(
					    "--transfo_xl_dataset_file",
					    default="",
					    type=str,
					    help="An optional dataset file to be converted in a vocabulary.",
					)
					__A        =    parser.parse_args()
					convert_transfo_xl_checkpoint_to_pytorch(
					    args.tf_checkpoint_path,
					    args.transfo_xl_config_file,
					    args.pytorch_dump_folder_path,
					    args.transfo_xl_dataset_file,
					)


