

"""simple docstring"""






import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
						from transformers import LlamaTokenizerFast
except ImportError as e:
						warnings.warn(e)
						warnings.warn(
						    '''The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'''
						)
						lowerCamelCase__					:	Dict				    =	None
lowerCamelCase__					:	List[str]				    =	{
    '''7B''': 1_10_08,
    '''13B''': 1_38_24,
    '''30B''': 1_79_20,
    '''65B''': 2_20_16,
    '''70B''': 2_86_72,
}
lowerCamelCase__					:	List[str]				    =	{
    '''7B''': 1,
    '''7Bf''': 1,
    '''13B''': 2,
    '''13Bf''': 2,
    '''30B''': 4,
    '''65B''': 8,
    '''70B''': 8,
    '''70Bf''': 8,
}
def    UpperCamelCase       (      _lowerCAmelCase  :    str,	_lowerCAmelCase  :    Dict=1,	_lowerCAmelCase  :    str=256			)					->      Tuple:
	return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3			)			) + multiple_of - 1) // multiple_of)
def    UpperCamelCase       (      _lowerCAmelCase  :    Any			)					->      Optional[Any]:




	with open(_UpperCamelCase,	"""r"""			) as f:
		return json.load(_UpperCamelCase			)
def    UpperCamelCase       (      _lowerCAmelCase  :    str,	_lowerCAmelCase  :    Optional[Any]			)					->      str:




	with open(_UpperCamelCase,	"""w"""			) as f:
		json.dump(_UpperCamelCase,	_UpperCamelCase			)
def    UpperCamelCase       (      _lowerCAmelCase  :    Any,	_lowerCAmelCase  :    Any,	_lowerCAmelCase  :    Any,	_lowerCAmelCase  :    Optional[Any]=True			)					->      List[str]:
	os.makedirs(_UpperCamelCase,	exist_ok=_UpperCamelCase			)
	_UpperCAmelCase     :				str          =       os.path.join(_UpperCamelCase,	"""tmp"""			)
	os.makedirs(_UpperCamelCase,	exist_ok=_UpperCamelCase			)

	_UpperCAmelCase     :				Union[str, Any]          =       read_json(os.path.join(_UpperCamelCase,	"""params.json"""			)			)
	_UpperCAmelCase     :				Optional[Any]          =       NUM_SHARDS[model_size]
	_UpperCAmelCase     :				List[str]          =       params["n_layers"]
	_UpperCAmelCase     :				int          =       params["n_heads"]
	_UpperCAmelCase     :				int          =       n_heads // num_shards
	_UpperCAmelCase     :				Any          =       params["dim"]
	_UpperCAmelCase     :				Any          =       dim // n_heads
	_UpperCAmelCase     :				Optional[int]          =       10000.0
	_UpperCAmelCase     :				Dict          =       1.0 / (base ** (torch.arange(0,	_UpperCamelCase,	2			).float() / dims_per_head))

	if "n_kv_heads" in params:
		_UpperCAmelCase     :				Tuple          =       params["n_kv_heads"]  # for GQA / MQA
		_UpperCAmelCase     :				str          =       n_heads_per_shard // num_key_value_heads
		_UpperCAmelCase     :				int          =       dim // num_key_value_heads
	else:  # compatibility with other checkpoints
		_UpperCAmelCase     :				Optional[int]          =       n_heads
		_UpperCAmelCase     :				List[Any]          =       n_heads_per_shard
		_UpperCAmelCase     :				List[Any]          =       dim

	# permute for sliced rotary
	def permute(_lowerCAmelCase  :    Optional[Any],	_lowerCAmelCase  :    Dict=n_heads,	_lowerCAmelCase  :    List[str]=dim,	_lowerCAmelCase  :    Dict=dim			):
		return w.view(_UpperCamelCase,	dima // n_heads // 2,	2,	_UpperCamelCase			).transpose(1,	2			).reshape(_UpperCamelCase,	_UpperCamelCase			)

	print(f'''Fetching all parameters from the checkpoint at {input_base_path}.'''			)
	# Load weights
	if model_size == "7B":
		# Not sharded
		# (The sharded implementation would also work, but this is simpler.)
		_UpperCAmelCase     :				Optional[Any]          =       torch.load(os.path.join(_UpperCamelCase,	"""consolidated.00.pth"""			),	map_location="""cpu"""			)
	else:
		# Sharded
		_UpperCAmelCase     :				Any          =       [
		    torch.load(os.path.join(_UpperCamelCase,	f'''consolidated.{i:02d}.pth'''			),	map_location="""cpu"""			)
		    for i in range(_UpperCamelCase			)
		]
	_UpperCAmelCase     :				Any          =       0
	_UpperCAmelCase     :				Tuple          =       {"weight_map": {}}
	for layer_i in range(_UpperCamelCase			):
		_UpperCAmelCase     :				List[Any]          =       f'''pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'''
		if model_size == "7B":
			# Unsharded
			_UpperCAmelCase     :				Optional[int]          =       {
			    f'''model.layers.{layer_i}.self_attn.q_proj.weight''': permute(
			        loaded[f'''layers.{layer_i}.attention.wq.weight''']			),
			    f'''model.layers.{layer_i}.self_attn.k_proj.weight''': permute(
			        loaded[f'''layers.{layer_i}.attention.wk.weight''']			),
			    f'''model.layers.{layer_i}.self_attn.v_proj.weight''': loaded[f'''layers.{layer_i}.attention.wv.weight'''],
			    f'''model.layers.{layer_i}.self_attn.o_proj.weight''': loaded[f'''layers.{layer_i}.attention.wo.weight'''],
			    f'''model.layers.{layer_i}.mlp.gate_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w1.weight'''],
			    f'''model.layers.{layer_i}.mlp.down_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w2.weight'''],
			    f'''model.layers.{layer_i}.mlp.up_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w3.weight'''],
			    f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[f'''layers.{layer_i}.attention_norm.weight'''],
			    f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[f'''layers.{layer_i}.ffn_norm.weight'''],
			}
		else:
			# Sharded
			# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
			# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
			# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

			_UpperCAmelCase     :				Optional[Any]          =       {
			    f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[0][
			        f'''layers.{layer_i}.attention_norm.weight'''
			    ].clone(),
			    f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[0][
			        f'''layers.{layer_i}.ffn_norm.weight'''
			    ].clone(),
			}
			_UpperCAmelCase     :				Dict          =       permute(
			    torch.cat(
			        [
			            loaded[i][f'''layers.{layer_i}.attention.wq.weight'''].view(_UpperCamelCase,	_UpperCamelCase,	_UpperCamelCase			)
			            for i in range(_UpperCamelCase			)
			        ],	dim=0,	).reshape(_UpperCamelCase,	_UpperCamelCase			)			)
			_UpperCAmelCase     :				int          =       permute(
			    torch.cat(
			        [
			            loaded[i][f'''layers.{layer_i}.attention.wk.weight'''].view(
			                _UpperCamelCase,	_UpperCamelCase,	_UpperCamelCase			)
			            for i in range(_UpperCamelCase			)
			        ],	dim=0,	).reshape(_UpperCamelCase,	_UpperCamelCase			),	_UpperCamelCase,	_UpperCamelCase,	_UpperCamelCase,	)
			_UpperCAmelCase     :				List[str]          =       torch.cat(
			    [
			        loaded[i][f'''layers.{layer_i}.attention.wv.weight'''].view(
			            _UpperCamelCase,	_UpperCamelCase,	_UpperCamelCase			)
			        for i in range(_UpperCamelCase			)
			    ],	dim=0,	).reshape(_UpperCamelCase,	_UpperCamelCase			)

			_UpperCAmelCase     :				Optional[Any]          =       torch.cat(
			    [loaded[i][f'''layers.{layer_i}.attention.wo.weight'''] for i in range(_UpperCamelCase			)],	dim=1			)
			_UpperCAmelCase     :				int          =       torch.cat(
			    [loaded[i][f'''layers.{layer_i}.feed_forward.w1.weight'''] for i in range(_UpperCamelCase			)],	dim=0			)
			_UpperCAmelCase     :				List[str]          =       torch.cat(
			    [loaded[i][f'''layers.{layer_i}.feed_forward.w2.weight'''] for i in range(_UpperCamelCase			)],	dim=1			)
			_UpperCAmelCase     :				List[Any]          =       torch.cat(
			    [loaded[i][f'''layers.{layer_i}.feed_forward.w3.weight'''] for i in range(_UpperCamelCase			)],	dim=0			)

		_UpperCAmelCase     :				str          =       inv_freq
		for k, v in state_dict.items():
			_UpperCAmelCase     :				str          =       filename
			param_count += v.numel()
		torch.save(_UpperCamelCase,	os.path.join(_UpperCamelCase,	_UpperCamelCase			)			)

	_UpperCAmelCase     :				Tuple          =       f'''pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'''
	if model_size == "7B":
		# Unsharded
		_UpperCAmelCase     :				Optional[int]          =       {
		    "model.embed_tokens.weight": loaded["tok_embeddings.weight"],
		    "model.norm.weight": loaded["norm.weight"],
		    "lm_head.weight": loaded["output.weight"],
		}
	else:
		_UpperCAmelCase     :				Union[str, Any]          =       {
		    "model.norm.weight": loaded[0]["norm.weight"],
		    "model.embed_tokens.weight": torch.cat(
		        [loaded[i]["""tok_embeddings.weight"""] for i in range(_UpperCamelCase			)],	dim=1			),
		    "lm_head.weight": torch.cat([loaded[i]["""output.weight"""] for i in range(_UpperCamelCase			)],	dim=0			),
		}

	for k, v in state_dict.items():
		_UpperCAmelCase     :				Tuple          =       filename
		param_count += v.numel()
	torch.save(_UpperCamelCase,	os.path.join(_UpperCamelCase,	_UpperCamelCase			)			)

	# Write configs
	_UpperCAmelCase     :				Dict          =       {"total_size": param_count * 2}
	write_json(_UpperCamelCase,	os.path.join(_UpperCamelCase,	"""pytorch_model.bin.index.json"""			)			)
	_UpperCAmelCase     :				Optional[Any]          =       params["ffn_dim_multiplier"] if "ffn_dim_multiplier" in params else 1
	_UpperCAmelCase     :				List[Any]          =       params["multiple_of"] if "multiple_of" in params else 256
	_UpperCAmelCase     :				List[Any]          =       LlamaConfig(
	    hidden_size=_UpperCamelCase,	intermediate_size=compute_intermediate_size(_UpperCamelCase,	_UpperCamelCase,	_UpperCamelCase			),	num_attention_heads=params["""n_heads"""],	num_hidden_layers=params["""n_layers"""],	rms_norm_eps=params["""norm_eps"""],	num_key_value_heads=_UpperCamelCase,	)
	config.save_pretrained(_UpperCamelCase			)

	# Make space so we can load the model properly now.
	del state_dict
	del loaded
	gc.collect()

	print("""Loading the checkpoint in a Llama model."""			)
	_UpperCAmelCase     :				int          =       LlamaForCausalLM.from_pretrained(_UpperCamelCase,	torch_dtype=torch.floataa,	low_cpu_mem_usage=_UpperCamelCase			)
	# Avoid saving this as part of the config.
	del model.config._name_or_path

	print("""Saving in the Transformers format."""			)
	model.save_pretrained(_UpperCamelCase,	safe_serialization=_UpperCamelCase			)
	shutil.rmtree(_UpperCamelCase			)
def    UpperCamelCase       (      _lowerCAmelCase  :    Dict,	_lowerCAmelCase  :    List[str]			)					->      str:
	_UpperCAmelCase     :				Optional[Any]          =       LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
	print(f'''Saving a {tokenizer_class.__name__} to {tokenizer_path}.'''			)
	_UpperCAmelCase     :				Any          =       tokenizer_class(_UpperCamelCase			)
	tokenizer.save_pretrained(_UpperCamelCase			)
def    UpperCamelCase       (      )					->      Optional[Any]:
	_UpperCAmelCase     :				Optional[Any]          =       argparse.ArgumentParser()
	parser.add_argument(
	    """--input_dir""",	help="""Location of LLaMA weights, which contains tokenizer.model and model folders""",	)
	parser.add_argument(
	    """--model_size""",	choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""],	)
	parser.add_argument(
	    """--output_dir""",	help="""Location to write HF model and tokenizer""",	)
	parser.add_argument("""--safe_serialization""",	type=_UpperCamelCase,	help="""Whether or not to save using `safetensors`."""			)
	_UpperCAmelCase     :				str          =       parser.parse_args()
	if args.model_size != "tokenizer_only":
		write_model(
		    model_path=args.output_dir,	input_base_path=os.path.join(args.input_dir,	args.model_size			),	model_size=args.model_size,	safe_serialization=args.safe_serialization,	)
	_UpperCAmelCase     :				Tuple          =       os.path.join(args.input_dir,	"""tokenizer.model"""			)
	write_tokenizer(args.output_dir,	_UpperCamelCase			)


if __name__ == "__main__":
						main()













'''simple docstring'''
import argparse

import torch

from transformers import (
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForAudioFrameClassification,
    WavaVecaForSequenceClassification,
    WavaVecaForXVector,
    logging,
)


logging.set_verbosity_info()
UpperCamelCase__			=      logging.get_logger(__name__)




def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase				):


   """simple docstring"""




   lowercase_						:     Optional[Any]							     =  WavaVecaForSequenceClassification.from_pretrained(_UpperCamelCase    ,		config=_UpperCamelCase				)
   lowercase_						:     Optional[int]							     =  downstream_dict["projector.weight"]
   lowercase_						:     str							     =  downstream_dict["projector.bias"]
   lowercase_						:     int							     =  downstream_dict["model.post_net.linear.weight"]
   lowercase_						:     Optional[Any]							     =  downstream_dict["model.post_net.linear.bias"]
   return model




def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase				):


   """simple docstring"""




   lowercase_						:     Tuple							     =  WavaVecaForAudioFrameClassification.from_pretrained(_UpperCamelCase    ,		config=_UpperCamelCase				)
   lowercase_						:     Any							     =  downstream_dict["model.linear.weight"]
   lowercase_						:     List[str]							     =  downstream_dict["model.linear.bias"]
   return model




def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase				):


   """simple docstring"""




   lowercase_						:     Any							     =  WavaVecaForXVector.from_pretrained(_UpperCamelCase    ,		config=_UpperCamelCase				)
   lowercase_						:     str							     =  downstream_dict["connector.weight"]
   lowercase_						:     List[str]							     =  downstream_dict["connector.bias"]
   for i, kernel_size in enumerate(hf_config.tdnn_kernel				):
      lowercase_						:     Union[str, Any]							     =  downstream_dict[
          F"""model.framelevel_feature_extractor.module.{i}.kernel.weight"""
      ]
      lowercase_						:     Dict							     =  downstream_dict[F"""model.framelevel_feature_extractor.module.{i}.kernel.bias"""]

   lowercase_						:     Dict							     =  downstream_dict["model.utterancelevel_feature_extractor.linear1.weight"]
   lowercase_						:     List[Any]							     =  downstream_dict["model.utterancelevel_feature_extractor.linear1.bias"]
   lowercase_						:     Dict							     =  downstream_dict["model.utterancelevel_feature_extractor.linear2.weight"]
   lowercase_						:     Dict							     =  downstream_dict["model.utterancelevel_feature_extractor.linear2.bias"]
   lowercase_						:     Optional[Any]							     =  downstream_dict["objective.W"]
   return model




@torch.no_grad()
def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase				):


   """simple docstring"""




   lowercase_						:     Tuple							     =  torch.load(_UpperCamelCase    ,		map_location="cpu"				)

   lowercase_						:     Dict							     =  checkpoint["Downstream"]

   lowercase_						:     Optional[Any]							     =  WavaVecaConfig.from_pretrained(_UpperCamelCase				)
   lowercase_						:     Dict							     =  WavaVecaFeatureExtractor.from_pretrained(
       _UpperCamelCase    ,		return_attention_mask=_UpperCamelCase    ,		do_normalize=_UpperCamelCase				)

   lowercase_						:     Dict							     =  hf_config.architectures[0]
   if arch.endswith("ForSequenceClassification"				):
      lowercase_						:     Any							     =  convert_classification(_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase				)
   elif arch.endswith("ForAudioFrameClassification"				):
      lowercase_						:     Optional[int]							     =  convert_diarization(_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase				)
   elif arch.endswith("ForXVector"				):
      lowercase_						:     List[Any]							     =  convert_xvector(_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase				)
   else:
      raise NotImplementedError(F"""S3PRL weights conversion is not supported for {arch}"""				)

   if hf_config.use_weighted_layer_sum:
      lowercase_						:     List[str]							     =  checkpoint["Featurizer"]["weights"]

   hf_feature_extractor.save_pretrained(_UpperCamelCase				)
   hf_model.save_pretrained(_UpperCamelCase				)


if __name__ == "__main__":
 UpperCamelCase__			=      argparse.ArgumentParser()
 parser.add_argument(
     '--base_model_name', default=None, type=str, help='Name of the huggingface pretrained base model.'
 )
 parser.add_argument('--config_path', default=None, type=str, help='Path to the huggingface classifier config.')
 parser.add_argument('--checkpoint_path', default=None, type=str, help='Path to the s3prl checkpoint.')
 parser.add_argument('--model_dump_path', default=None, type=str, help='Path to the final converted model.')
 UpperCamelCase__			=      parser.parse_args()
 convert_saprl_checkpoint(args.base_model_name, args.config_path, args.checkpoint_path, args.model_dump_path)




