




# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce
def 		a_							( __lowerCAmelCase				):
  return (torch.arange(state.num_processes				) + 1.0 + (state.num_processes * state.process_index)).to(state.device				)
def 		a_							( __lowerCAmelCase				):
  lowerCAmelCase__										=      create_tensor(__lowerCAmelCase				)
  lowerCAmelCase__										=      gather(__lowerCAmelCase				)
  assert gathered_tensor.tolist() == list(range(1   , state.num_processes**2 + 1				)				)
def 		a_							( __lowerCAmelCase				):
  lowerCAmelCase__										=      [state.process_index]
  lowerCAmelCase__										=      gather_object(__lowerCAmelCase				)
  assert len(__lowerCAmelCase				) == state.num_processes, F"""{gathered_obj}, {len(__lowerCAmelCase				)} != {state.num_processes}"""
  assert gathered_obj == list(range(state.num_processes				)				), F"""{gathered_obj} != {list(range(state.num_processes				)				)}"""
def 		a_							( __lowerCAmelCase				):
  lowerCAmelCase__										=      create_tensor(__lowerCAmelCase				)
  lowerCAmelCase__										=      broadcast(__lowerCAmelCase				)
  assert broadcasted_tensor.shape == torch.Size([state.num_processes]				)
  assert broadcasted_tensor.tolist() == list(range(1   , state.num_processes + 1				)				)
def 		a_							( __lowerCAmelCase				):
  # We need to pad the tensor with one more element if we are the main process
  # to ensure that we can pad
  if state.is_main_process:
    lowerCAmelCase__										=      torch.arange(state.num_processes + 1				).to(state.device				)
  else:
    lowerCAmelCase__										=      torch.arange(state.num_processes				).to(state.device				)
  lowerCAmelCase__										=      pad_across_processes(__lowerCAmelCase				)
  assert padded_tensor.shape == torch.Size([state.num_processes + 1]				)
  if not state.is_main_process:
    assert padded_tensor.tolist() == list(range(0   , state.num_processes				)				) + [0]
def 		a_							( __lowerCAmelCase				):
  # For now runs on only two processes
  if state.num_processes != 2:
    return
  lowerCAmelCase__										=      create_tensor(__lowerCAmelCase				)
  lowerCAmelCase__										=      reduce(__lowerCAmelCase   , '''sum'''				)
  lowerCAmelCase__										=      torch.tensor([4.0, 6]				).to(state.device				)
  assert torch.allclose(__lowerCAmelCase   , __lowerCAmelCase				), F"""{reduced_tensor} != {truth_tensor}"""
def 		a_							( __lowerCAmelCase				):
  # For now runs on only two processes
  if state.num_processes != 2:
    return
  lowerCAmelCase__										=      create_tensor(__lowerCAmelCase				)
  lowerCAmelCase__										=      reduce(__lowerCAmelCase   , '''mean'''				)
  lowerCAmelCase__										=      torch.tensor([2.0, 3]				).to(state.device				)
  assert torch.allclose(__lowerCAmelCase   , __lowerCAmelCase				), F"""{reduced_tensor} != {truth_tensor}"""
def 		a_							( __lowerCAmelCase				):
  # For xla_spawn (TPUs)
  main()
def 		a_							( ):
  lowerCAmelCase__										=      PartialState()
  state.print(F"""State: {state}"""				)
  state.print('''testing gather'''				)
  test_gather(__lowerCAmelCase				)
  state.print('''testing gather_object'''				)
  test_gather_object(__lowerCAmelCase				)
  state.print('''testing broadcast'''				)
  test_broadcast(__lowerCAmelCase				)
  state.print('''testing pad_across_processes'''				)
  test_pad_across_processes(__lowerCAmelCase				)
  state.print('''testing reduce_sum'''				)
  test_reduce_sum(__lowerCAmelCase				)
  state.print('''testing reduce_mean'''				)
  test_reduce_mean(__lowerCAmelCase				)


if __name__ == "__main__":
      main()







import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
__magic_name__  :  Tuple					=       data_utils.TransfoXLTokenizer
__magic_name__  :  int					=       data_utils.TransfoXLCorpus
__magic_name__  :  List[str]					=       data_utils
__magic_name__  :  str					=       data_utils
def 		a_							( __lowerCAmelCase   , __lowerCAmelCase   , __lowerCAmelCase   , __lowerCAmelCase				):
  if transfo_xl_dataset_file:
    # Convert a pre-processed corpus (see original TensorFlow repo)
    with open(__lowerCAmelCase   , '''rb'''				) as fp:
      lowerCAmelCase__										=      pickle.load(__lowerCAmelCase   , encoding='''latin1'''				)
    # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
    lowerCAmelCase__										=      pytorch_dump_folder_path + '''/''' + VOCAB_FILES_NAMES['''pretrained_vocab_file''']
    print(F"""Save vocabulary to {pytorch_vocab_dump_path}"""				)
    lowerCAmelCase__										=      corpus.vocab.__dict__
    torch.save(__lowerCAmelCase   , __lowerCAmelCase				)

    lowerCAmelCase__										=      corpus.__dict__
    corpus_dict_no_vocab.pop('''vocab'''   , __lowerCAmelCase				)
    lowerCAmelCase__										=      pytorch_dump_folder_path + '''/''' + CORPUS_NAME
    print(F"""Save dataset to {pytorch_dataset_dump_path}"""				)
    torch.save(__lowerCAmelCase   , __lowerCAmelCase				)

  if tf_checkpoint_path:
    # Convert a pre-trained TensorFlow model
    lowerCAmelCase__										=      os.path.abspath(__lowerCAmelCase				)
    lowerCAmelCase__										=      os.path.abspath(__lowerCAmelCase				)

    print(F"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."""				)
    # Initialise PyTorch model
    if transfo_xl_config_file == "":
      lowerCAmelCase__										=      TransfoXLConfig()
    else:
      lowerCAmelCase__										=      TransfoXLConfig.from_json_file(__lowerCAmelCase				)
    print(F"""Building PyTorch model from configuration: {config}"""				)
    lowerCAmelCase__										=      TransfoXLLMHeadModel(__lowerCAmelCase				)

    lowerCAmelCase__										=      load_tf_weights_in_transfo_xl(__lowerCAmelCase   , __lowerCAmelCase   , __lowerCAmelCase				)
    # Save pytorch-model
    lowerCAmelCase__										=      os.path.join(__lowerCAmelCase   , __lowerCAmelCase				)
    lowerCAmelCase__										=      os.path.join(__lowerCAmelCase   , __lowerCAmelCase				)
    print(F"""Save PyTorch model to {os.path.abspath(__lowerCAmelCase				)}"""				)
    torch.save(model.state_dict()   , __lowerCAmelCase				)
    print(F"""Save configuration file to {os.path.abspath(__lowerCAmelCase				)}"""				)
    with open(__lowerCAmelCase   , '''w'''   , encoding='''utf-8'''				) as f:
      f.write(config.to_json_string()				)


if __name__ == "__main__":
      __magic_name__  :  int					=       argparse.ArgumentParser()
      parser.add_argument(
          """--pytorch_dump_folder_path""",
          default=None,
          type=str,
          required=True,
          help="""Path to the folder to store the PyTorch model or dataset/vocab.""",
      )
      parser.add_argument(
          """--tf_checkpoint_path""",
          default="""""",
          type=str,
          help="""An optional path to a TensorFlow checkpoint path to be converted.""",
      )
      parser.add_argument(
          """--transfo_xl_config_file""",
          default="""""",
          type=str,
          help=(
              """An optional config json file corresponding to the pre-trained BERT model. \n"""
              """This specifies the model architecture."""
          ),
      )
      parser.add_argument(
          """--transfo_xl_dataset_file""",
          default="""""",
          type=str,
          help="""An optional dataset file to be converted in a vocabulary.""",
      )
      __magic_name__  :  Optional[Any]					=       parser.parse_args()
      convert_transfo_xl_checkpoint_to_pytorch(
          args.tf_checkpoint_path,
          args.transfo_xl_config_file,
          args.pytorch_dump_folder_path,
          args.transfo_xl_dataset_file,
      )

