






"""simple docstring"""


import argparse
import json
import os

import torch
from torch import nn

from transformers import NllbMoeConfig, NllbMoeModel
from transformers.modeling_utils import dtype_byte_size
from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME



def 					lowerCAmelCase				(    __UpperCamelCase ):




  """simple docstring"""







  __A									= [
      '''encoder.version''',
      '''decoder.version''',
      '''model.encoder.version''',
      '''model.decoder.version''',
      '''decoder.output_projection.weight''',
      '''_float_tensor''',
      '''encoder.embed_positions._float_tensor''',
      '''decoder.embed_positions._float_tensor''',
  ]
  for k in ignore_keys:
    state_dict.pop(__UpperCamelCase		,       __UpperCamelCase )



def 					lowerCAmelCase				(    __UpperCamelCase ):




  """simple docstring"""







  __A			,    __A									= emb.weight.shape
  __A									= nn.Linear(__UpperCamelCase		,       __UpperCamelCase		,       bias=__UpperCamelCase )
  __A									= emb.weight.data
  return lin_layer



def 					lowerCAmelCase				(    __UpperCamelCase		,       __UpperCamelCase=None ):




  """simple docstring"""







  __A									= {}
  for old_key in state_dict.keys():
    __A									= old_key
    if "moe_layer.experts." in key:
      if expert_idx is not None:
        __A									= key.replace('''moe_layer.experts.0'''		,       f'ffn.experts.expert_{expert_idx}' )
      else:
        __A									= key.replace('''moe_layer.experts.'''		,       '''ffn.experts.expert_''' )
    if "gate" in key:
      __A									= key.replace('''.moe_layer.gate.wg'''		,       '''.ffn.router.classifier''' )
    if "fc2" and "experts" not in key:
      __A									= key.replace('''.fc2.'''		,       '''.ffn.fc2.''' )
    if "fc1" and "experts" not in key:
      __A									= key.replace('''.fc1.'''		,       '''.ffn.fc1.''' )
    if ".encoder_attn." in key:
      __A									= key.replace('''.encoder_attn.'''		,       '''.cross_attention.''' )
    if "encoder_attn_layer_norm" in key:
      __A									= key.replace('''encoder_attn_layer_norm'''		,       '''cross_attention_layer_norm''' )
    if "final_layer_norm" in key:
      __A									= key.replace('''final_layer_norm'''		,       '''ff_layer_norm''' )
    __A									= state_dict[old_key]
  return new_dict



def 					lowerCAmelCase				(    __UpperCamelCase		,       __UpperCamelCase		,       __UpperCamelCase		,       __UpperCamelCase		,       __UpperCamelCase = WEIGHTS_NAME ):




  """simple docstring"""







  __A									= []
  __A									= 0
  os.makedirs(__UpperCamelCase		,       exist_ok=__UpperCamelCase )

  for expert in range(__UpperCamelCase ):
    __A									= switch_checkpoint_path + f'-rank-{expert}.pt'
    if os.path.isfile(__UpperCamelCase ):
      __A									= torch.load(__UpperCamelCase )['''model''']
      remove_ignore_keys_(__UpperCamelCase )
      __A									= rename_fairseq_keys(__UpperCamelCase		,       __UpperCamelCase )
      __A									= os.path.join(
          __UpperCamelCase		,       weights_name.replace('''.bin'''		,       f'-{len(__UpperCamelCase )+1:05d}-of-???.bin' ) )
      torch.save(__UpperCamelCase		,       __UpperCamelCase )
      sharded_state_dicts.append(expert_state.keys() )
      total_size += sum([value.numel() for key, value in expert_state.items()] ) * dtype_byte_size(
          expert_state[list(__UpperCamelCase )[0]].dtype )

    # Add the last block
  __A									= os.path.join(__UpperCamelCase		,       weights_name.replace('''.bin'''		,       f'-{len(__UpperCamelCase )+1:05d}-of-???.bin' ) )
  __A									= torch.load(switch_checkpoint_path + '''-shared.pt''' )['''model''']
  remove_ignore_keys_(__UpperCamelCase )
  __A									= rename_fairseq_keys(__UpperCamelCase		,       __UpperCamelCase )
  __A									= shared_weights['''decoder.embed_tokens.weight''']
  sharded_state_dicts.append(shared_weights.keys() )

  # If we only have the shared weights (dummy model/experts saved on the same file)
  if len(__UpperCamelCase ) == 1:
    __A									= os.path.join(__UpperCamelCase		,       __UpperCamelCase )
    torch.save(__UpperCamelCase		,       __UpperCamelCase )
    return {weights_name: sharded_state_dicts[0]}, None
  else:
    torch.save(__UpperCamelCase		,       __UpperCamelCase )
  # Otherwise, let's build the index
  __A									= {}
  for idx, shard in enumerate(__UpperCamelCase ):
    __A									= weights_name.replace('''.bin'''		,       f'-{idx+1:05d}-of-{len(__UpperCamelCase ):05d}.bin' )
    __A									= os.path.join(__UpperCamelCase		,       weights_name.replace('''.bin'''		,       f'-{idx+1:05d}-of-???.bin' ) )
    os.rename(__UpperCamelCase		,       os.path.join(__UpperCamelCase		,       __UpperCamelCase ) )
    for key in shard:
      __A									= shard_file

    # Add the metadata
  __A									= {'''total_size''': total_size}
  __A									= {'''metadata''': metadata, '''weight_map''': weight_map}

  with open(os.path.join(__UpperCamelCase		,       __UpperCamelCase )		,       '''w'''		,       encoding='''utf-8''' ) as f:
    __A									= json.dumps(__UpperCamelCase		,       indent=2		,       sort_keys=__UpperCamelCase ) + '''\n'''
    f.write(__UpperCamelCase )

  return metadata, index


if __name__ == "__main__":
       lowercase_				     =     argparse.ArgumentParser()
       # Required parameters
       parser.add_argument(
           '--nllb_moe_checkpoint_path',
           default='/home/arthur_huggingface_co/fairseq/weights/checkpoints/model_moe_54b/checkpoint_2_300000',
           type=str,
           required=False,
           help='Path to a directory containing a folder per layer. Follows the original Google format.',
       )
       parser.add_argument('--dtype', default='float32', type=str, required=False, help='dtype of the saved model')
       parser.add_argument(
           '--pytorch_dump_folder_path',
           default='/home/arthur_huggingface_co/fairseq/weights/checkpoints/hf-converted-moe-54b',
           type=str,
           required=False,
           help='Path to the output pytorch model.',
       )
       lowercase_				     =     parser.parse_args()
       lowercase_     ,		lowercase_				     =     shard_on_the_fly(
           args.nllb_moe_checkpoint_path,
           args.pytorch_dump_folder_path,
           128,
           args.dtype,
       )

       lowercase_				     =     NllbMoeConfig.from_pretrained(
           'facebook/nllb-200-3.3B', encoder_sparse_step=4, decoder_sparse_step=4, num_experts=128
       )
       config.save_pretrained(args.pytorch_dump_folder_path)
       lowercase_				     =     NllbMoeModel.from_pretrained(args.pytorch_dump_folder_path)
       print('Done')
       model.save_pretrained(args.pytorch_dump_folder_path)












"""simple docstring"""


import os
import sys
from contextlib import contextmanager


# Windows only
if os.name == "nt":
       import ctypes
       import msvcrt  # noqa


       class 					snake_case		(							ctypes.Structure						):


          '''simple docstring'''

          A_						:	List[str]						 = [("size", ctypes.c_int), ("visible", ctypes.c_byte)]



def 					lowerCAmelCase				(    ):




  """simple docstring"""







  if os.name == "nt":
    __A									= CursorInfo()
    __A									= ctypes.windll.kernelaa.GetStdHandle(-1_1 )
    ctypes.windll.kernelaa.GetConsoleCursorInfo(__UpperCamelCase		,       ctypes.byref(__UpperCamelCase ) )
    __A									= False
    ctypes.windll.kernelaa.SetConsoleCursorInfo(__UpperCamelCase		,       ctypes.byref(__UpperCamelCase ) )
  elif os.name == "posix":
    sys.stdout.write('''\033[?25l''' )
    sys.stdout.flush()



def 					lowerCAmelCase				(    ):




  """simple docstring"""







  if os.name == "nt":
    __A									= CursorInfo()
    __A									= ctypes.windll.kernelaa.GetStdHandle(-1_1 )
    ctypes.windll.kernelaa.GetConsoleCursorInfo(__UpperCamelCase		,       ctypes.byref(__UpperCamelCase ) )
    __A									= True
    ctypes.windll.kernelaa.SetConsoleCursorInfo(__UpperCamelCase		,       ctypes.byref(__UpperCamelCase ) )
  elif os.name == "posix":
    sys.stdout.write('''\033[?25h''' )
    sys.stdout.flush()





@contextmanager
def 					lowerCAmelCase				(    ):




  """simple docstring"""







  try:
    hide_cursor()
    yield
  finally:
    show_cursor()




