


"""simple docstring"""

import argparse
import json
import os

import torch
from torch import nn

from transformers import NllbMoeConfig, NllbMoeModel
from transformers.modeling_utils import dtype_byte_size
from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME


def lowerCAmelCase_   (					UpperCamelCase__						:			Tuple			):






 """simple docstring"""







 __lowercase						=						[
     """encoder.version""",
     """decoder.version""",
     """model.encoder.version""",
     """model.decoder.version""",
     """decoder.output_projection.weight""",
     """_float_tensor""",
     """encoder.embed_positions._float_tensor""",
     """decoder.embed_positions._float_tensor""",
 ]
 for k in ignore_keys:
  state_dict.pop(UpperCamelCase__					, UpperCamelCase__			)


def lowerCAmelCase_   (					UpperCamelCase__						:			str			):






 """simple docstring"""







 __lowercase				,		__lowercase						=						emb.weight.shape
 __lowercase						=						nn.Linear(UpperCamelCase__					, UpperCamelCase__					, bias=UpperCamelCase__			)
 __lowercase						=						emb.weight.data
 return lin_layer


def lowerCAmelCase_   (					UpperCamelCase__						:			Union[str, Any]					, UpperCamelCase__						:			List[str]=None			):






 """simple docstring"""







 __lowercase						=						{}
 for old_key in state_dict.keys():
  __lowercase						=						old_key
  if "moe_layer.experts." in key:
   if expert_idx is not None:
    __lowercase						=						key.replace("""moe_layer.experts.0"""					, f'''ffn.experts.expert_{expert_idx}'''			)
   else:
    __lowercase						=						key.replace("""moe_layer.experts."""					, """ffn.experts.expert_"""			)
  if "gate" in key:
   __lowercase						=						key.replace(""".moe_layer.gate.wg"""					, """.ffn.router.classifier"""			)
  if "fc2" and "experts" not in key:
   __lowercase						=						key.replace(""".fc2."""					, """.ffn.fc2."""			)
  if "fc1" and "experts" not in key:
   __lowercase						=						key.replace(""".fc1."""					, """.ffn.fc1."""			)
  if ".encoder_attn." in key:
   __lowercase						=						key.replace(""".encoder_attn."""					, """.cross_attention."""			)
  if "encoder_attn_layer_norm" in key:
   __lowercase						=						key.replace("""encoder_attn_layer_norm"""					, """cross_attention_layer_norm"""			)
  if "final_layer_norm" in key:
   __lowercase						=						key.replace("""final_layer_norm"""					, """ff_layer_norm"""			)
  __lowercase						=						state_dict[old_key]
 return new_dict


def lowerCAmelCase_   (					UpperCamelCase__						:			Any					, UpperCamelCase__						:			Optional[Any]					, UpperCamelCase__						:			List[Any]					, UpperCamelCase__						:			Tuple					, UpperCamelCase__						:			str = WEIGHTS_NAME			):






 """simple docstring"""







 __lowercase						=						[]
 __lowercase						=						0
 os.makedirs(UpperCamelCase__					, exist_ok=UpperCamelCase__			)

 for expert in range(UpperCamelCase__			):
  __lowercase						=						switch_checkpoint_path + f'''-rank-{expert}.pt'''
  if os.path.isfile(UpperCamelCase__			):
   __lowercase						=						torch.load(UpperCamelCase__			)["""model"""]
   remove_ignore_keys_(UpperCamelCase__			)
   __lowercase						=						rename_fairseq_keys(UpperCamelCase__					, UpperCamelCase__			)
   __lowercase						=						os.path.join(
       UpperCamelCase__					, weights_name.replace(""".bin"""					, f'''-{len(UpperCamelCase__			)+1:05d}-of-???.bin'''			)			)
   torch.save(UpperCamelCase__					, UpperCamelCase__			)
   sharded_state_dicts.append(expert_state.keys()			)
   total_size += sum([value.numel() for key, value in expert_state.items()]			) * dtype_byte_size(
       expert_state[list(UpperCamelCase__			)[0]].dtype			)

    # Add the last block
 __lowercase						=						os.path.join(UpperCamelCase__					, weights_name.replace(""".bin"""					, f'''-{len(UpperCamelCase__			)+1:05d}-of-???.bin'''			)			)
 __lowercase						=						torch.load(switch_checkpoint_path + """-shared.pt"""			)["""model"""]
 remove_ignore_keys_(UpperCamelCase__			)
 __lowercase						=						rename_fairseq_keys(UpperCamelCase__					, UpperCamelCase__			)
 __lowercase						=						shared_weights["""decoder.embed_tokens.weight"""]
 sharded_state_dicts.append(shared_weights.keys()			)

 # If we only have the shared weights (dummy model/experts saved on the same file)
 if len(UpperCamelCase__			) == 1:
  __lowercase						=						os.path.join(UpperCamelCase__					, UpperCamelCase__			)
  torch.save(UpperCamelCase__					, UpperCamelCase__			)
  return {weights_name: sharded_state_dicts[0]}, None
 else:
  torch.save(UpperCamelCase__					, UpperCamelCase__			)
 # Otherwise, let's build the index
 __lowercase						=						{}
 for idx, shard in enumerate(UpperCamelCase__			):
  __lowercase						=						weights_name.replace(""".bin"""					, f'''-{idx+1:05d}-of-{len(UpperCamelCase__			):05d}.bin'''			)
  __lowercase						=						os.path.join(UpperCamelCase__					, weights_name.replace(""".bin"""					, f'''-{idx+1:05d}-of-???.bin'''			)			)
  os.rename(UpperCamelCase__					, os.path.join(UpperCamelCase__					, UpperCamelCase__			)			)
  for key in shard:
   __lowercase						=						shard_file

    # Add the metadata
 __lowercase						=						{"""total_size""": total_size}
 __lowercase						=						{"""metadata""": metadata, """weight_map""": weight_map}

 with open(os.path.join(UpperCamelCase__					, UpperCamelCase__			)					, """w"""					, encoding="""utf-8"""			) as f:
  __lowercase						=						json.dumps(UpperCamelCase__					, indent=2					, sort_keys=UpperCamelCase__			) + """\n"""
  f.write(UpperCamelCase__			)

 return metadata, index


if __name__ == "__main__":
       UpperCAmelCase__					=argparse.ArgumentParser()
       # Required parameters
       parser.add_argument(
           "--nllb_moe_checkpoint_path",
           default="/home/arthur_huggingface_co/fairseq/weights/checkpoints/model_moe_54b/checkpoint_2_300000",
           type=str,
           required=False,
           help="Path to a directory containing a folder per layer. Follows the original Google format.",
       )
       parser.add_argument("--dtype", default="float32", type=str, required=False, help="dtype of the saved model")
       parser.add_argument(
           "--pytorch_dump_folder_path",
           default="/home/arthur_huggingface_co/fairseq/weights/checkpoints/hf-converted-moe-54b",
           type=str,
           required=False,
           help="Path to the output pytorch model.",
       )
       UpperCAmelCase__					=parser.parse_args()
       UpperCAmelCase__				,   UpperCAmelCase__					=shard_on_the_fly(
           args.nllb_moe_checkpoint_path,
           args.pytorch_dump_folder_path,
           128,
           args.dtype,
       )

       UpperCAmelCase__					=NllbMoeConfig.from_pretrained(
           "facebook/nllb-200-3.3B", encoder_sparse_step=4, decoder_sparse_step=4, num_experts=128
       )
       config.save_pretrained(args.pytorch_dump_folder_path)
       UpperCAmelCase__					=NllbMoeModel.from_pretrained(args.pytorch_dump_folder_path)
       print("Done")
       model.save_pretrained(args.pytorch_dump_folder_path)




"""simple docstring"""

def lowerCAmelCase_   (					UpperCamelCase__						:			Union[str, Any]					, UpperCamelCase__						:			Union[str, Any]					, UpperCamelCase__						:			int			):






 """simple docstring"""







 if n == 0:
  return 1

 elif n % 2 == 1:
  return (binary_exponentiation(UpperCamelCase__					, n - 1					, UpperCamelCase__			) * a) % mod

 else:
  __lowercase						=						binary_exponentiation(UpperCamelCase__					, n / 2					, UpperCamelCase__			)
  return (b * b) % mod


# a prime number
UpperCAmelCase__					=701

UpperCAmelCase__					=10_0000_0000
UpperCAmelCase__					=10

# using binary exponentiation function, O(log(p)):
print((a / b) % p == (a * binary_exponentiation(b, p - 2, p)) % p)

print((a / b) % p == (a * b ** (p - 2)) % p)
