
"""simple docstring"""







import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
lowerCamelCase__   					=		data_utils.TransfoXLTokenizer
lowerCamelCase__   					=		data_utils.TransfoXLCorpus
lowerCamelCase__   					=		data_utils
lowerCamelCase__   					=		data_utils
def     __lowerCAmelCase				(_UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase		):
	if transfo_xl_dataset_file:
		# Convert a pre-processed corpus (see original TensorFlow repo)
		with open(_UpperCamelCase       ,  'rb'		) as fp:
			__lowerCAmelCase			:						Dict									=      pickle.load(_UpperCamelCase       ,  encoding='latin1'		)
		# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
		__lowerCAmelCase			:						Union[str, Any]									=      pytorch_dump_folder_path + '/' + VOCAB_FILES_NAMES['pretrained_vocab_file']
		print(F"Save vocabulary to {pytorch_vocab_dump_path}"		)
		__lowerCAmelCase			:						Optional[int]									=      corpus.vocab.__dict__
		torch.save(_UpperCamelCase       ,  _UpperCamelCase		)

		__lowerCAmelCase			:						Tuple									=      corpus.__dict__
		corpus_dict_no_vocab.pop('vocab'       ,  _UpperCamelCase		)
		__lowerCAmelCase			:						Tuple									=      pytorch_dump_folder_path + '/' + CORPUS_NAME
		print(F"Save dataset to {pytorch_dataset_dump_path}"		)
		torch.save(_UpperCamelCase       ,  _UpperCamelCase		)

	if tf_checkpoint_path:
		# Convert a pre-trained TensorFlow model
		__lowerCAmelCase			:						str									=      os.path.abspath(_UpperCamelCase		)
		__lowerCAmelCase			:						Any									=      os.path.abspath(_UpperCamelCase		)

		print(F"Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."		)
		# Initialise PyTorch model
		if transfo_xl_config_file == "":
			__lowerCAmelCase			:						List[Any]									=      TransfoXLConfig()
		else:
			__lowerCAmelCase			:						List[Any]									=      TransfoXLConfig.from_json_file(_UpperCamelCase		)
		print(F"Building PyTorch model from configuration: {config}"		)
		__lowerCAmelCase			:						Union[str, Any]									=      TransfoXLLMHeadModel(_UpperCamelCase		)

		__lowerCAmelCase			:						Tuple									=      load_tf_weights_in_transfo_xl(_UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase		)
		# Save pytorch-model
		__lowerCAmelCase			:						str									=      os.path.join(_UpperCamelCase       ,  _UpperCamelCase		)
		__lowerCAmelCase			:						Union[str, Any]									=      os.path.join(_UpperCamelCase       ,  _UpperCamelCase		)
		print(F"Save PyTorch model to {os.path.abspath(_UpperCamelCase		)}"		)
		torch.save(model.state_dict()       ,  _UpperCamelCase		)
		print(F"Save configuration file to {os.path.abspath(_UpperCamelCase		)}"		)
		with open(_UpperCamelCase       ,  'w'       ,  encoding='utf-8'		) as f:
			f.write(config.to_json_string()		)


if __name__ == "__main__":
							lowerCamelCase__   					=		argparse.ArgumentParser()
							parser.add_argument(
							    """--pytorch_dump_folder_path""",
							    default=None,
							    type=str,
							    required=True,
							    help="""Path to the folder to store the PyTorch model or dataset/vocab.""",
							)
							parser.add_argument(
							    """--tf_checkpoint_path""",
							    default="""""",
							    type=str,
							    help="""An optional path to a TensorFlow checkpoint path to be converted.""",
							)
							parser.add_argument(
							    """--transfo_xl_config_file""",
							    default="""""",
							    type=str,
							    help=(
							        """An optional config json file corresponding to the pre-trained BERT model. \n"""
							        """This specifies the model architecture."""
							    ),
							)
							parser.add_argument(
							    """--transfo_xl_dataset_file""",
							    default="""""",
							    type=str,
							    help="""An optional dataset file to be converted in a vocabulary.""",
							)
							lowerCamelCase__   					=		parser.parse_args()
							convert_transfo_xl_checkpoint_to_pytorch(
							    args.tf_checkpoint_path,
							    args.transfo_xl_config_file,
							    args.pytorch_dump_folder_path,
							    args.transfo_xl_dataset_file,
							)

'''simple docstring'''







import inspect
import os
import unittest
from pathlib import Path

import torch

import accelerate
from accelerate.test_utils import execute_subprocess_async
from accelerate.test_utils.testing import run_command







class UpperCAmelCase__	( unittest.TestCase	):





  """simple docstring"""

  __UpperCAmelCase				:				Optional[Any]   =					inspect.getfile(accelerate.test_utils	)
  __UpperCAmelCase				:				List[str]   =					os.path.sep.join(mod_file.split(os.path.sep	)[:-1] + ['''scripts''', '''test_cli.py''']	)

  __UpperCAmelCase				:				Dict   =					['''accelerate''', '''launch''']
  __UpperCAmelCase				:				Dict   =					Path.home() / '''.cache/huggingface/accelerate'''
  __UpperCAmelCase				:				Dict   =					'''default_config.yaml'''
  __UpperCAmelCase				:				Optional[Any]   =					config_folder / config_file
  __UpperCAmelCase				:				Dict   =					config_folder / '''_default_config.yaml'''

  __UpperCAmelCase				:				Any   =					Path('''tests/test_configs'''	)


  @classmethod
  def   __lowercase		(		cls      :     int       ):


     '''simple docstring'''






     if cls.config_path.is_file():
        cls.config_path.rename(cls.changed_path       )


  @classmethod
  def   __lowercase		(		cls      :     List[Any]       ):


     '''simple docstring'''






     if cls.changed_path.is_file():
        cls.changed_path.rename(cls.config_path       )


  def   __lowercase		(		self      :     Optional[int]       ):


     '''simple docstring'''






     _a				:	Dict			=   self.base_cmd
     if torch.cuda.is_available() and (torch.cuda.device_count() > 1):
        cmd += ["--multi_gpu"]
     execute_subprocess_async(cmd + [self.test_file_path]							,env=os.environ.copy()       )


  def   __lowercase		(		self      :     Optional[Any]       ):


     '''simple docstring'''






     for config in sorted(self.test_config_path.glob('**/*.yaml'       )       ):
        with self.subTest(config_file=_a       ):
           execute_subprocess_async(
               self.base_cmd + ['--config_file', str(_a       ), self.test_file_path]							,env=os.environ.copy()       )


  def   __lowercase		(		self      :     Optional[int]       ):


     '''simple docstring'''






     execute_subprocess_async(['accelerate', 'test']							,env=os.environ.copy()       )







class UpperCAmelCase__	( unittest.TestCase	):





  """simple docstring"""

  __UpperCAmelCase				:				Optional[Any]   =					'''test-tpu'''
  __UpperCAmelCase				:				Any   =					'''us-central1-a'''
  __UpperCAmelCase				:				List[Any]   =					'''ls'''
  __UpperCAmelCase				:				Any   =					['''accelerate''', '''tpu-config''']
  __UpperCAmelCase				:				Dict   =					'''cd /usr/share'''
  __UpperCAmelCase				:				Any   =					'''tests/test_samples/test_command_file.sh'''
  __UpperCAmelCase				:				List[Any]   =					'''Running gcloud compute tpus tpu-vm ssh'''


  def   __lowercase		(		self      :     Dict       ):


     '''simple docstring'''






     _a				:	Optional[Any]			=   run_command(
         self.cmd
         + ['--command', self.command, '--tpu_zone', self.tpu_zone, '--tpu_name', self.tpu_name, '--debug']							,return_stdout=_a							,)
     self.assertIn(
         F"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all"""							,_a							,)


  def   __lowercase		(		self      :     List[str]       ):


     '''simple docstring'''






     _a				:	Any			=   run_command(
         self.cmd
         + [
             '--config_file',
             'tests/test_configs/0_12_0.yaml',
             '--command',
             self.command,
             '--tpu_zone',
             self.tpu_zone,
             '--tpu_name',
             self.tpu_name,
             '--debug',
         ]							,return_stdout=_a							,)
     self.assertIn(
         F"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all"""							,_a							,)


  def   __lowercase		(		self      :     List[str]       ):


     '''simple docstring'''






     _a				:	Optional[int]			=   run_command(
         self.cmd + ['--config_file', 'tests/test_configs/latest.yaml', '--debug']							,return_stdout=_a       )
     self.assertIn(
         F"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo \"hello world\"; echo \"this is a second command\" --worker all"""							,_a							,)


  def   __lowercase		(		self      :     int       ):


     '''simple docstring'''






     _a				:	Optional[Any]			=   run_command(
         self.cmd + ['--config_file', 'tests/test_configs/latest.yaml', '--command', self.command, '--debug']							,return_stdout=_a							,)
     self.assertIn(
         F"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all"""							,_a							,)


  def   __lowercase		(		self      :     str       ):


     '''simple docstring'''






     _a				:	List[str]			=   run_command(
         self.cmd
         + [
             '--config_file',
             'tests/test_configs/latest.yaml',
             '--command',
             self.command,
             '--command',
             'echo "Hello World"',
             '--debug',
         ]							,return_stdout=_a							,)
     self.assertIn(
         F"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls; echo \"Hello World\" --worker all"""							,_a							,)


  def   __lowercase		(		self      :     Union[str, Any]       ):


     '''simple docstring'''






     _a				:	Any			=   run_command(
         self.cmd
         + ['--config_file', 'tests/test_configs/latest.yaml', '--command_file', self.command_file, '--debug']							,return_stdout=_a							,)
     self.assertIn(
         F"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo \"hello world\"; echo \"this is a second command\" --worker all"""							,_a							,)


  def   __lowercase		(		self      :     Union[str, Any]       ):


     '''simple docstring'''






     _a				:	Union[str, Any]			=   run_command(
         self.cmd
         + [
             '--config_file',
             'tests/test_configs/0_12_0.yaml',
             '--command_file',
             self.command_file,
             '--tpu_zone',
             self.tpu_zone,
             '--tpu_name',
             self.tpu_name,
             '--debug',
         ]							,return_stdout=_a							,)
     self.assertIn(
         F"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo \"hello world\"; echo \"this is a second command\" --worker all"""							,_a							,)


  def   __lowercase		(		self      :     Any       ):


     '''simple docstring'''






     _a				:	Optional[int]			=   run_command(
         self.cmd + ['--config_file', 'tests/test_configs/latest.yaml', '--install_accelerate', '--debug']							,return_stdout=_a							,)
     self.assertIn(
         F"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate -U; echo \"hello world\"; echo \"this is a second command\" --worker all"""							,_a							,)


  def   __lowercase		(		self      :     List[str]       ):


     '''simple docstring'''






     _a				:	Optional[int]			=   run_command(
         self.cmd
         + [
             '--config_file',
             'tests/test_configs/latest.yaml',
             '--install_accelerate',
             '--accelerate_version',
             '12.0.0',
             '--debug',
         ]							,return_stdout=_a							,)
     self.assertIn(
         F"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate==12.0.0; echo \"hello world\"; echo \"this is a second command\" --worker all"""							,_a							,)



