



import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
lowerCAmelCase__		:     List[Any]       						=data_utils.TransfoXLTokenizer
lowerCAmelCase__		:     Optional[int]       						=data_utils.TransfoXLCorpus
lowerCAmelCase__		:     Tuple       						=data_utils
lowerCAmelCase__		:     Tuple       						=data_utils






def 	__lowercase   ( a__							,   a__							,   a__							,   a__   )				-> Union[str, Any]:
							if transfo_xl_dataset_file:
														# Convert a pre-processed corpus (see original TensorFlow repo)
														with open(__lowerCamelCase							,   'rb'   ) as fp:
																					__SCREAMING_SNAKE_CASE									=					pickle.load(__lowerCamelCase							,   encoding='latin1'   )
														# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
														__SCREAMING_SNAKE_CASE									=					pytorch_dump_folder_path + """/""" + VOCAB_FILES_NAMES["""pretrained_vocab_file"""]
														print(f"""Save vocabulary to {pytorch_vocab_dump_path}"""   )
														__SCREAMING_SNAKE_CASE									=					corpus.vocab.__dict__
														torch.save(__lowerCamelCase							,   __lowerCamelCase   )

														__SCREAMING_SNAKE_CASE									=					corpus.__dict__
														corpus_dict_no_vocab.pop('vocab'							,   __lowerCamelCase   )
														__SCREAMING_SNAKE_CASE									=					pytorch_dump_folder_path + """/""" + CORPUS_NAME
														print(f"""Save dataset to {pytorch_dataset_dump_path}"""   )
														torch.save(__lowerCamelCase							,   __lowerCamelCase   )

							if tf_checkpoint_path:
														# Convert a pre-trained TensorFlow model
														__SCREAMING_SNAKE_CASE									=					os.path.abspath(__lowerCamelCase   )
														__SCREAMING_SNAKE_CASE									=					os.path.abspath(__lowerCamelCase   )

														print(f"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."""   )
														# Initialise PyTorch model
														if transfo_xl_config_file == "":
																					__SCREAMING_SNAKE_CASE									=					TransfoXLConfig()
														else:
																					__SCREAMING_SNAKE_CASE									=					TransfoXLConfig.from_json_file(__lowerCamelCase   )
														print(f"""Building PyTorch model from configuration: {config}"""   )
														__SCREAMING_SNAKE_CASE									=					TransfoXLLMHeadModel(__lowerCamelCase   )

														__SCREAMING_SNAKE_CASE									=					load_tf_weights_in_transfo_xl(__lowerCamelCase							,   __lowerCamelCase							,   __lowerCamelCase   )
														# Save pytorch-model
														__SCREAMING_SNAKE_CASE									=					os.path.join(__lowerCamelCase							,   __lowerCamelCase   )
														__SCREAMING_SNAKE_CASE									=					os.path.join(__lowerCamelCase							,   __lowerCamelCase   )
														print(f"""Save PyTorch model to {os.path.abspath(__lowerCamelCase   )}"""   )
														torch.save(model.state_dict()							,   __lowerCamelCase   )
														print(f"""Save configuration file to {os.path.abspath(__lowerCamelCase   )}"""   )
														with open(__lowerCamelCase							,   'w'							,   encoding='utf-8'   ) as f:
																					f.write(config.to_json_string()   )


if __name__ == "__main__":
			lowerCAmelCase__		:     Any       						=argparse.ArgumentParser()
			parser.add_argument(
			    '''--pytorch_dump_folder_path''',
			    default=None,
			    type=str,
			    required=True,
			    help='''Path to the folder to store the PyTorch model or dataset/vocab.''',
			)
			parser.add_argument(
			    '''--tf_checkpoint_path''',
			    default='''''',
			    type=str,
			    help='''An optional path to a TensorFlow checkpoint path to be converted.''',
			)
			parser.add_argument(
			    '''--transfo_xl_config_file''',
			    default='''''',
			    type=str,
			    help=(
			        '''An optional config json file corresponding to the pre-trained BERT model. \n'''
			        '''This specifies the model architecture.'''
			    ),
			)
			parser.add_argument(
			    '''--transfo_xl_dataset_file''',
			    default='''''',
			    type=str,
			    help='''An optional dataset file to be converted in a vocabulary.''',
			)
			lowerCAmelCase__		:     List[Any]       						=parser.parse_args()
			convert_transfo_xl_checkpoint_to_pytorch(
			    args.tf_checkpoint_path,
			    args.transfo_xl_config_file,
			    args.pytorch_dump_folder_path,
			    args.transfo_xl_dataset_file,
			)














from abc import ABC, abstractmethod
from typing import Optional, Union

from .. import Dataset, DatasetDict, Features, IterableDataset, IterableDatasetDict, NamedSplit
from ..utils.typing import NestedDataStructureLike, PathLike
class     a    ( lowercase__   ):



      """simple docstring"""


      def __init__(      self					:			Any		,      __lowercase					:			Optional[NestedDataStructureLike[PathLike]] = None		,      __lowercase					:			Optional[NamedSplit] = None		,      __lowercase					:			Optional[Features] = None		,      __lowercase					:			str = None		,      __lowercase					:			bool = False		,      __lowercase					:			bool = False		,      __lowercase					:			Optional[int] = None		,      **__lowercase					:			List[str]		,      )							->      Tuple:
        __UpperCAmelCase							:	Any           =  path_or_paths
        __UpperCAmelCase							:	Dict           =  split if split or isinstance(__lowercase		,      __lowercase					) else """train"""
        __UpperCAmelCase							:	Optional[int]           =  features
        __UpperCAmelCase							:	str           =  cache_dir
        __UpperCAmelCase							:	str           =  keep_in_memory
        __UpperCAmelCase							:	Optional[int]           =  streaming
        __UpperCAmelCase							:	Dict           =  num_proc
        __UpperCAmelCase							:	Tuple           =  kwargs




      @abstractmethod
      def 	UpperCAmelCase      (      self					:			List[Any]					)							->      Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]:
        pass


class     a    ( lowercase__   ):



      """simple docstring"""


      def __init__(      self					:			List[str]		,      __lowercase					:			Optional[Features] = None		,      __lowercase					:			str = None		,      __lowercase					:			bool = False		,      __lowercase					:			bool = False		,      __lowercase					:			Optional[int] = None		,      **__lowercase					:			Optional[Any]		,      )							->      Optional[int]:
        __UpperCAmelCase							:	Optional[Any]           =  features
        __UpperCAmelCase							:	str           =  cache_dir
        __UpperCAmelCase							:	Optional[int]           =  keep_in_memory
        __UpperCAmelCase							:	Dict           =  streaming
        __UpperCAmelCase							:	Optional[Any]           =  num_proc
        __UpperCAmelCase							:	Union[str, Any]           =  kwargs




      @abstractmethod
      def 	UpperCAmelCase      (      self					:			List[str]					)							->      Union[Dataset, IterableDataset]:
        pass


