


import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
   from transformers import LlamaTokenizerFast
except ImportError as e:
   warnings.warn(e)
   warnings.warn(
       '''The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'''
   )
   _snake_case             =     None
_snake_case             =     {
    '''7B''': 1_10_08,
    '''13B''': 1_38_24,
    '''30B''': 1_79_20,
    '''65B''': 2_20_16,
    '''70B''': 2_86_72,
}
_snake_case             =     {
    '''7B''': 1,
    '''7Bf''': 1,
    '''13B''': 2,
    '''13Bf''': 2,
    '''30B''': 4,
    '''65B''': 8,
    '''70B''': 8,
    '''70Bf''': 8,
}






def 		lowercase_(  SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_=1      ,						SCREAMING_SNAKE_CASE_=256	):


  '''simple docstring'''

  return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3	)	) + multiple_of - 1) // multiple_of)






def 		lowercase_(  SCREAMING_SNAKE_CASE_	):


  '''simple docstring'''

  with open(SCREAMING_SNAKE_CASE_      ,						"r"	) as f:
    return json.load(SCREAMING_SNAKE_CASE_	)






def 		lowercase_(  SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_	):


  '''simple docstring'''

  with open(SCREAMING_SNAKE_CASE_      ,						"w"	) as f:
    json.dump(SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_	)






def 		lowercase_(  SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_=True	):


  '''simple docstring'''

  os.makedirs(SCREAMING_SNAKE_CASE_      ,						exist_ok=SCREAMING_SNAKE_CASE_	)
  lowerCamelCase :					List[Any]    =				os.path.join(SCREAMING_SNAKE_CASE_      ,						"tmp"	)
  os.makedirs(SCREAMING_SNAKE_CASE_      ,						exist_ok=SCREAMING_SNAKE_CASE_	)

  lowerCamelCase :					List[str]    =				read_json(os.path.join(SCREAMING_SNAKE_CASE_      ,						"params.json"	)	)
  lowerCamelCase :					Tuple    =				NUM_SHARDS[model_size]
  lowerCamelCase :					List[str]    =				params["n_layers"]
  lowerCamelCase :					int    =				params["n_heads"]
  lowerCamelCase :					List[str]    =				n_heads // num_shards
  lowerCamelCase :					Tuple    =				params["dim"]
  lowerCamelCase :					Optional[int]    =				dim // n_heads
  lowerCamelCase :					Dict    =				10000.0
  lowerCamelCase :					str    =				1.0 / (base ** (torch.arange(0      ,						SCREAMING_SNAKE_CASE_      ,						2	).float() / dims_per_head))

  if "n_kv_heads" in params:
    lowerCamelCase :					Union[str, Any]    =				params["n_kv_heads"]  # for GQA / MQA
    lowerCamelCase :					Tuple    =				n_heads_per_shard // num_key_value_heads
    lowerCamelCase :					str    =				dim // num_key_value_heads
  else:  # compatibility with other checkpoints
    lowerCamelCase :					Union[str, Any]    =				n_heads
    lowerCamelCase :					int    =				n_heads_per_shard
    lowerCamelCase :					Union[str, Any]    =				dim

  # permute for sliced rotary
  def permute(SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_=n_heads      ,						SCREAMING_SNAKE_CASE_=dim      ,						SCREAMING_SNAKE_CASE_=dim	):
    return w.view(SCREAMING_SNAKE_CASE_      ,						dima // n_heads // 2      ,						2      ,						SCREAMING_SNAKE_CASE_	).transpose(1      ,						2	).reshape(SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_	)

  print(f"""Fetching all parameters from the checkpoint at {input_base_path}."""	)
  # Load weights
  if model_size == "7B":
    # Not sharded
    # (The sharded implementation would also work, but this is simpler.)
    lowerCamelCase :					List[str]    =				torch.load(os.path.join(SCREAMING_SNAKE_CASE_      ,						"consolidated.00.pth"	)      ,						map_location="cpu"	)
  else:
    # Sharded
    lowerCamelCase :					Dict    =				[
        torch.load(os.path.join(SCREAMING_SNAKE_CASE_      ,						f"""consolidated.{i:02d}.pth"""	)      ,						map_location="cpu"	)
        for i in range(SCREAMING_SNAKE_CASE_	)
    ]
  lowerCamelCase :					Any    =				0
  lowerCamelCase :					Any    =				{"weight_map": {}}
  for layer_i in range(SCREAMING_SNAKE_CASE_	):
    lowerCamelCase :					Optional[int]    =				f"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
    if model_size == "7B":
      # Unsharded
      lowerCamelCase :					Optional[int]    =				{
          f"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
              loaded[f"""layers.{layer_i}.attention.wq.weight"""]	),
          f"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
              loaded[f"""layers.{layer_i}.attention.wk.weight"""]	),
          f"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[f"""layers.{layer_i}.attention.wv.weight"""],
          f"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[f"""layers.{layer_i}.attention.wo.weight"""],
          f"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w1.weight"""],
          f"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w2.weight"""],
          f"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w3.weight"""],
          f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[f"""layers.{layer_i}.attention_norm.weight"""],
          f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[f"""layers.{layer_i}.ffn_norm.weight"""],
      }
    else:
      # Sharded
      # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
      # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
      # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

      lowerCamelCase :					Dict    =				{
          f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
              f"""layers.{layer_i}.attention_norm.weight"""
          ].clone(),
          f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
              f"""layers.{layer_i}.ffn_norm.weight"""
          ].clone(),
      }
      lowerCamelCase :					Optional[Any]    =				permute(
          torch.cat(
              [
                  loaded[i][f"""layers.{layer_i}.attention.wq.weight"""].view(SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_	)
                  for i in range(SCREAMING_SNAKE_CASE_	)
              ]      ,						dim=0      ,						).reshape(SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_	)	)
      lowerCamelCase :					Union[str, Any]    =				permute(
          torch.cat(
              [
                  loaded[i][f"""layers.{layer_i}.attention.wk.weight"""].view(
                      SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_	)
                  for i in range(SCREAMING_SNAKE_CASE_	)
              ]      ,						dim=0      ,						).reshape(SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_	)      ,						SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_      ,						)
      lowerCamelCase :					Optional[int]    =				torch.cat(
          [
              loaded[i][f"""layers.{layer_i}.attention.wv.weight"""].view(
                  SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_	)
              for i in range(SCREAMING_SNAKE_CASE_	)
          ]      ,						dim=0      ,						).reshape(SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_	)

      lowerCamelCase :					int    =				torch.cat(
          [loaded[i][f"""layers.{layer_i}.attention.wo.weight"""] for i in range(SCREAMING_SNAKE_CASE_	)]      ,						dim=1	)
      lowerCamelCase :					Tuple    =				torch.cat(
          [loaded[i][f"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(SCREAMING_SNAKE_CASE_	)]      ,						dim=0	)
      lowerCamelCase :					List[str]    =				torch.cat(
          [loaded[i][f"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(SCREAMING_SNAKE_CASE_	)]      ,						dim=1	)
      lowerCamelCase :					Optional[Any]    =				torch.cat(
          [loaded[i][f"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(SCREAMING_SNAKE_CASE_	)]      ,						dim=0	)

    lowerCamelCase :					List[Any]    =				inv_freq
    for k, v in state_dict.items():
      lowerCamelCase :					Dict    =				filename
      param_count += v.numel()
    torch.save(SCREAMING_SNAKE_CASE_      ,						os.path.join(SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_	)	)

  lowerCamelCase :					List[str]    =				f"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
  if model_size == "7B":
    # Unsharded
    lowerCamelCase :					Optional[int]    =				{
        "model.embed_tokens.weight": loaded["tok_embeddings.weight"],
        "model.norm.weight": loaded["norm.weight"],
        "lm_head.weight": loaded["output.weight"],
    }
  else:
    lowerCamelCase :					Any    =				{
        "model.norm.weight": loaded[0]["norm.weight"],
        "model.embed_tokens.weight": torch.cat(
            [loaded[i]["tok_embeddings.weight"] for i in range(SCREAMING_SNAKE_CASE_	)]      ,						dim=1	),
        "lm_head.weight": torch.cat([loaded[i]["output.weight"] for i in range(SCREAMING_SNAKE_CASE_	)]      ,						dim=0	),
    }

  for k, v in state_dict.items():
    lowerCamelCase :					List[Any]    =				filename
    param_count += v.numel()
  torch.save(SCREAMING_SNAKE_CASE_      ,						os.path.join(SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_	)	)

  # Write configs
  lowerCamelCase :					Optional[int]    =				{"total_size": param_count * 2}
  write_json(SCREAMING_SNAKE_CASE_      ,						os.path.join(SCREAMING_SNAKE_CASE_      ,						"pytorch_model.bin.index.json"	)	)
  lowerCamelCase :					Tuple    =				params["ffn_dim_multiplier"] if "ffn_dim_multiplier" in params else 1
  lowerCamelCase :					Dict    =				params["multiple_of"] if "multiple_of" in params else 256
  lowerCamelCase :					Tuple    =				LlamaConfig(
      hidden_size=SCREAMING_SNAKE_CASE_      ,						intermediate_size=compute_intermediate_size(SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_	)      ,						num_attention_heads=params["n_heads"]      ,						num_hidden_layers=params["n_layers"]      ,						rms_norm_eps=params["norm_eps"]      ,						num_key_value_heads=SCREAMING_SNAKE_CASE_      ,						)
  config.save_pretrained(SCREAMING_SNAKE_CASE_	)

  # Make space so we can load the model properly now.
  del state_dict
  del loaded
  gc.collect()

  print("Loading the checkpoint in a Llama model."	)
  lowerCamelCase :					List[Any]    =				LlamaForCausalLM.from_pretrained(SCREAMING_SNAKE_CASE_      ,						torch_dtype=torch.floataa      ,						low_cpu_mem_usage=SCREAMING_SNAKE_CASE_	)
  # Avoid saving this as part of the config.
  del model.config._name_or_path

  print("Saving in the Transformers format."	)
  model.save_pretrained(SCREAMING_SNAKE_CASE_      ,						safe_serialization=SCREAMING_SNAKE_CASE_	)
  shutil.rmtree(SCREAMING_SNAKE_CASE_	)






def 		lowercase_(  SCREAMING_SNAKE_CASE_      ,						SCREAMING_SNAKE_CASE_	):


  '''simple docstring'''

  lowerCamelCase :					Union[str, Any]    =				LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
  print(f"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""	)
  lowerCamelCase :					int    =				tokenizer_class(SCREAMING_SNAKE_CASE_	)
  tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_	)






def 		lowercase_(  ):


  '''simple docstring'''

  lowerCamelCase :					List[Any]    =				argparse.ArgumentParser()
  parser.add_argument(
      "--input_dir"      ,						help="Location of LLaMA weights, which contains tokenizer.model and model folders"      ,						)
  parser.add_argument(
      "--model_size"      ,						choices=["7B", "7Bf", "13B", "13Bf", "30B", "65B", "70B", "70Bf", "tokenizer_only"]      ,						)
  parser.add_argument(
      "--output_dir"      ,						help="Location to write HF model and tokenizer"      ,						)
  parser.add_argument("--safe_serialization"      ,						type=SCREAMING_SNAKE_CASE_      ,						help="Whether or not to save using `safetensors`."	)
  lowerCamelCase :					Union[str, Any]    =				parser.parse_args()
  if args.model_size != "tokenizer_only":
    write_model(
        model_path=args.output_dir      ,						input_base_path=os.path.join(args.input_dir      ,						args.model_size	)      ,						model_size=args.model_size      ,						safe_serialization=args.safe_serialization      ,						)
  lowerCamelCase :					int    =				os.path.join(args.input_dir      ,						"tokenizer.model"	)
  write_tokenizer(args.output_dir      ,						SCREAMING_SNAKE_CASE_	)


if __name__ == "__main__":
   main()







def 		lowercase_(  SCREAMING_SNAKE_CASE_	):


  '''simple docstring'''

  for i in range(len(SCREAMING_SNAKE_CASE_	) - 1      ,						0      ,						-1	):
    lowerCamelCase :					Tuple    =				False

    for j in range(SCREAMING_SNAKE_CASE_      ,						0      ,						-1	):
      if unsorted[j] < unsorted[j - 1]:
        lowerCamelCase    ,	lowerCamelCase :					int    =				unsorted[j - 1], unsorted[j]
        lowerCamelCase :					Optional[int]    =				True

    for j in range(SCREAMING_SNAKE_CASE_	):
      if unsorted[j] > unsorted[j + 1]:
        lowerCamelCase    ,	lowerCamelCase :					Union[str, Any]    =				unsorted[j + 1], unsorted[j]
        lowerCamelCase :					Optional[Any]    =				True

    if not swapped:
      break
  return unsorted


if __name__ == "__main__":
   import doctest

   doctest.testmod()
   _snake_case             =     input('''Enter numbers separated by a comma:\n''').strip()
   _snake_case             =     [int(item) for item in user_input.split(''',''')]
   print(f'''{cocktail_shaker_sort(unsorted) = }''')



