




"""simple docstring"""


import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
     from transformers import LlamaTokenizerFast
except ImportError as e:
     warnings.warn(e)
     warnings.warn(
         '''The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'''
     )
     A_						=				None
A_						=				{
    '''7B''': 1_10_08,
    '''13B''': 1_38_24,
    '''30B''': 1_79_20,
    '''65B''': 2_20_16,
    '''70B''': 2_86_72,
}
A_						=				{
    '''7B''': 1,
    '''7Bf''': 1,
    '''13B''': 2,
    '''13Bf''': 2,
    '''30B''': 4,
    '''65B''': 8,
    '''70B''': 8,
    '''70Bf''': 8,
}


def 					UpperCAmelCase__       (snake_case__							:					Optional[int]		,   snake_case__							:					Optional[int]=1		,   snake_case__							:					List[str]=2_56    ):
  """simple docstring"""

  return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3    )    ) + multiple_of - 1) // multiple_of)


def 					UpperCAmelCase__       (snake_case__							:					List[Any]    ):
  """simple docstring"""

  with open(_UpperCAmelCase		,   """r"""    ) as f:
    return json.load(_UpperCAmelCase    )


def 					UpperCAmelCase__       (snake_case__							:					str		,   snake_case__							:					Optional[Any]    ):
  """simple docstring"""

  with open(_UpperCAmelCase		,   """w"""    ) as f:
    json.dump(_UpperCAmelCase		,   _UpperCAmelCase    )


def 					UpperCAmelCase__       (snake_case__							:					str		,   snake_case__							:					Optional[Any]		,   snake_case__							:					int		,   snake_case__							:					List[Any]=True    ):
  """simple docstring"""

  os.makedirs(_UpperCAmelCase		,   exist_ok=_UpperCAmelCase    )
  _snake_case							:		Union[str, Any]								=  os.path.join(_UpperCAmelCase		,   """tmp"""    )
  os.makedirs(_UpperCAmelCase		,   exist_ok=_UpperCAmelCase    )

  _snake_case							:		int								=  read_json(os.path.join(_UpperCAmelCase		,   """params.json"""    )    )
  _snake_case							:		int								=  NUM_SHARDS[model_size]
  _snake_case							:		Dict								=  params["""n_layers"""]
  _snake_case							:		Union[str, Any]								=  params["""n_heads"""]
  _snake_case							:		List[str]								=  n_heads // num_shards
  _snake_case							:		int								=  params["""dim"""]
  _snake_case							:		Union[str, Any]								=  dim // n_heads
  _snake_case							:		int								=  1_00_00.0
  _snake_case							:		str								=  1.0 / (base ** (torch.arange(0		,   _UpperCAmelCase		,   2    ).float() / dims_per_head))

  if "n_kv_heads" in params:
    _snake_case							:		str								=  params["""n_kv_heads"""]  # for GQA / MQA
    _snake_case							:		Optional[Any]								=  n_heads_per_shard // num_key_value_heads
    _snake_case							:		Optional[int]								=  dim // num_key_value_heads
  else:  # compatibility with other checkpoints
    _snake_case							:		str								=  n_heads
    _snake_case							:		Any								=  n_heads_per_shard
    _snake_case							:		str								=  dim

  # permute for sliced rotary
  def permute(snake_case__							:					Tuple		,   snake_case__							:					Optional[int]=n_heads		,   snake_case__							:					Optional[int]=dim		,   snake_case__							:					List[str]=dim    ):
    return w.view(_UpperCAmelCase		,   dima // n_heads // 2		,   2		,   _UpperCAmelCase    ).transpose(1		,   2    ).reshape(_UpperCAmelCase		,   _UpperCAmelCase    )

  print(F"Fetching all parameters from the checkpoint at {input_base_path}."    )
  # Load weights
  if model_size == "7B":
    # Not sharded
    # (The sharded implementation would also work, but this is simpler.)
    _snake_case							:		Any								=  torch.load(os.path.join(_UpperCAmelCase		,   """consolidated.00.pth"""    )		,   map_location="""cpu"""    )
  else:
    # Sharded
    _snake_case							:		List[Any]								=  [
        torch.load(os.path.join(_UpperCAmelCase		,   F"consolidated.{i:02d}.pth"    )		,   map_location="""cpu"""    )
        for i in range(_UpperCAmelCase    )
    ]
  _snake_case							:		Any								=  0
  _snake_case							:		Optional[int]								=  {"""weight_map""": {}}
  for layer_i in range(_UpperCAmelCase    ):
    _snake_case							:		List[str]								=  F"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"
    if model_size == "7B":
      # Unsharded
      _snake_case							:		List[str]								=  {
          F"model.layers.{layer_i}.self_attn.q_proj.weight": permute(
              loaded[F"layers.{layer_i}.attention.wq.weight"]    ),
          F"model.layers.{layer_i}.self_attn.k_proj.weight": permute(
              loaded[F"layers.{layer_i}.attention.wk.weight"]    ),
          F"model.layers.{layer_i}.self_attn.v_proj.weight": loaded[F"layers.{layer_i}.attention.wv.weight"],
          F"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[F"layers.{layer_i}.attention.wo.weight"],
          F"model.layers.{layer_i}.mlp.gate_proj.weight": loaded[F"layers.{layer_i}.feed_forward.w1.weight"],
          F"model.layers.{layer_i}.mlp.down_proj.weight": loaded[F"layers.{layer_i}.feed_forward.w2.weight"],
          F"model.layers.{layer_i}.mlp.up_proj.weight": loaded[F"layers.{layer_i}.feed_forward.w3.weight"],
          F"model.layers.{layer_i}.input_layernorm.weight": loaded[F"layers.{layer_i}.attention_norm.weight"],
          F"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[F"layers.{layer_i}.ffn_norm.weight"],
      }
    else:
      # Sharded
      # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
      # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
      # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

      _snake_case							:		Tuple								=  {
          F"model.layers.{layer_i}.input_layernorm.weight": loaded[0][
              F"layers.{layer_i}.attention_norm.weight"
          ].clone(),
          F"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][
              F"layers.{layer_i}.ffn_norm.weight"
          ].clone(),
      }
      _snake_case							:		str								=  permute(
          torch.cat(
              [
                  loaded[i][F"layers.{layer_i}.attention.wq.weight"].view(_UpperCAmelCase		,   _UpperCAmelCase		,   _UpperCAmelCase    )
                  for i in range(_UpperCAmelCase    )
              ]		,   dim=0		,   ).reshape(_UpperCAmelCase		,   _UpperCAmelCase    )    )
      _snake_case							:		Tuple								=  permute(
          torch.cat(
              [
                  loaded[i][F"layers.{layer_i}.attention.wk.weight"].view(
                      _UpperCAmelCase		,   _UpperCAmelCase		,   _UpperCAmelCase    )
                  for i in range(_UpperCAmelCase    )
              ]		,   dim=0		,   ).reshape(_UpperCAmelCase		,   _UpperCAmelCase    )		,   _UpperCAmelCase		,   _UpperCAmelCase		,   _UpperCAmelCase		,   )
      _snake_case							:		Any								=  torch.cat(
          [
              loaded[i][F"layers.{layer_i}.attention.wv.weight"].view(
                  _UpperCAmelCase		,   _UpperCAmelCase		,   _UpperCAmelCase    )
              for i in range(_UpperCAmelCase    )
          ]		,   dim=0		,   ).reshape(_UpperCAmelCase		,   _UpperCAmelCase    )

      _snake_case							:		List[str]								=  torch.cat(
          [loaded[i][F"layers.{layer_i}.attention.wo.weight"] for i in range(_UpperCAmelCase    )]		,   dim=1    )
      _snake_case							:		Union[str, Any]								=  torch.cat(
          [loaded[i][F"layers.{layer_i}.feed_forward.w1.weight"] for i in range(_UpperCAmelCase    )]		,   dim=0    )
      _snake_case							:		Tuple								=  torch.cat(
          [loaded[i][F"layers.{layer_i}.feed_forward.w2.weight"] for i in range(_UpperCAmelCase    )]		,   dim=1    )
      _snake_case							:		Union[str, Any]								=  torch.cat(
          [loaded[i][F"layers.{layer_i}.feed_forward.w3.weight"] for i in range(_UpperCAmelCase    )]		,   dim=0    )

    _snake_case							:		str								=  inv_freq
    for k, v in state_dict.items():
      _snake_case							:		Any								=  filename
      param_count += v.numel()
    torch.save(_UpperCAmelCase		,   os.path.join(_UpperCAmelCase		,   _UpperCAmelCase    )    )

  _snake_case							:		Union[str, Any]								=  F"pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"
  if model_size == "7B":
    # Unsharded
    _snake_case							:		List[str]								=  {
        """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
        """model.norm.weight""": loaded["""norm.weight"""],
        """lm_head.weight""": loaded["""output.weight"""],
    }
  else:
    _snake_case							:		int								=  {
        """model.norm.weight""": loaded[0]["""norm.weight"""],
        """model.embed_tokens.weight""": torch.cat(
            [loaded[i]["""tok_embeddings.weight"""] for i in range(_UpperCAmelCase    )]		,   dim=1    ),
        """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(_UpperCAmelCase    )]		,   dim=0    ),
    }

  for k, v in state_dict.items():
    _snake_case							:		Dict								=  filename
    param_count += v.numel()
  torch.save(_UpperCAmelCase		,   os.path.join(_UpperCAmelCase		,   _UpperCAmelCase    )    )

  # Write configs
  _snake_case							:		Tuple								=  {"""total_size""": param_count * 2}
  write_json(_UpperCAmelCase		,   os.path.join(_UpperCAmelCase		,   """pytorch_model.bin.index.json"""    )    )
  _snake_case							:		Optional[Any]								=  params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
  _snake_case							:		int								=  params["""multiple_of"""] if """multiple_of""" in params else 2_56
  _snake_case							:		List[Any]								=  LlamaConfig(
      hidden_size=_UpperCAmelCase		,   intermediate_size=compute_intermediate_size(_UpperCAmelCase		,   _UpperCAmelCase		,   _UpperCAmelCase    )		,   num_attention_heads=params["""n_heads"""]		,   num_hidden_layers=params["""n_layers"""]		,   rms_norm_eps=params["""norm_eps"""]		,   num_key_value_heads=_UpperCAmelCase		,   )
  config.save_pretrained(_UpperCAmelCase    )

  # Make space so we can load the model properly now.
  del state_dict
  del loaded
  gc.collect()

  print("""Loading the checkpoint in a Llama model."""    )
  _snake_case							:		Any								=  LlamaForCausalLM.from_pretrained(_UpperCAmelCase		,   torch_dtype=torch.floataa		,   low_cpu_mem_usage=_UpperCAmelCase    )
  # Avoid saving this as part of the config.
  del model.config._name_or_path

  print("""Saving in the Transformers format."""    )
  model.save_pretrained(_UpperCAmelCase		,   safe_serialization=_UpperCAmelCase    )
  shutil.rmtree(_UpperCAmelCase    )


def 					UpperCAmelCase__       (snake_case__							:					int		,   snake_case__							:					int    ):
  """simple docstring"""

  _snake_case							:		List[str]								=  LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
  print(F"Saving a {tokenizer_class.__name__} to {tokenizer_path}."    )
  _snake_case							:		List[Any]								=  tokenizer_class(_UpperCAmelCase    )
  tokenizer.save_pretrained(_UpperCAmelCase    )


def 					UpperCAmelCase__       ():
  """simple docstring"""

  _snake_case							:		List[str]								=  argparse.ArgumentParser()
  parser.add_argument(
      """--input_dir"""		,   help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""		,   )
  parser.add_argument(
      """--model_size"""		,   choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]		,   )
  parser.add_argument(
      """--output_dir"""		,   help="""Location to write HF model and tokenizer"""		,   )
  parser.add_argument("""--safe_serialization"""		,   type=_UpperCAmelCase		,   help="""Whether or not to save using `safetensors`."""    )
  _snake_case							:		Optional[Any]								=  parser.parse_args()
  if args.model_size != "tokenizer_only":
    write_model(
        model_path=args.output_dir		,   input_base_path=os.path.join(args.input_dir		,   args.model_size    )		,   model_size=args.model_size		,   safe_serialization=args.safe_serialization		,   )
  _snake_case							:		List[Any]								=  os.path.join(args.input_dir		,   """tokenizer.model"""    )
  write_tokenizer(args.output_dir		,   _UpperCAmelCase    )


if __name__ == "__main__":
     main()






'''simple docstring'''







from ...configuration_utils import PretrainedConfig
from ...utils import logging


A__:    Dict					    =       logging.get_logger(__name__)

A__:    Tuple					    =       {
    '''weiweishi/roc-bert-base-zh''': '''https://huggingface.co/weiweishi/roc-bert-base-zh/resolve/main/config.json''',
}







class A__       (						UpperCAmelCase__      ):
      __UpperCamelCase       :  Tuple	       =							"roc_bert"






      def __init__( self			:Optional[int]     ,						SCREAMING_SNAKE_CASE			:Tuple=3_0_5_2_2     ,						SCREAMING_SNAKE_CASE			:List[str]=7_6_8     ,						SCREAMING_SNAKE_CASE			:Dict=1_2     ,						SCREAMING_SNAKE_CASE			:List[str]=1_2     ,						SCREAMING_SNAKE_CASE			:Tuple=3_0_7_2     ,						SCREAMING_SNAKE_CASE			:List[Any]="gelu"     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]=0.1     ,						SCREAMING_SNAKE_CASE			:List[Any]=0.1     ,						SCREAMING_SNAKE_CASE			:int=5_1_2     ,						SCREAMING_SNAKE_CASE			:Optional[Any]=2     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]=0.02     ,						SCREAMING_SNAKE_CASE			:Optional[Any]=1e-12     ,						SCREAMING_SNAKE_CASE			:Any=True     ,						SCREAMING_SNAKE_CASE			:List[Any]=0     ,						SCREAMING_SNAKE_CASE			:Optional[int]="absolute"     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]=None     ,						SCREAMING_SNAKE_CASE			:List[Any]=True     ,						SCREAMING_SNAKE_CASE			:int=True     ,						SCREAMING_SNAKE_CASE			:Optional[int]=7_6_8     ,						SCREAMING_SNAKE_CASE			:Optional[Any]=9_1_0     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]=5_1_2     ,						SCREAMING_SNAKE_CASE			:str=2_4_8_5_8     ,						SCREAMING_SNAKE_CASE			:List[Any]=True     ,						**SCREAMING_SNAKE_CASE			:Tuple     ,						)       ->					Optional[int]:




             '''simple docstring'''
             _a    : List[str]     					=vocab_size
             _a    : List[str]     					=max_position_embeddings
             _a    : Optional[Any]     					=hidden_size
             _a    : List[Any]     					=num_hidden_layers
             _a    : List[str]     					=num_attention_heads
             _a    : int     					=intermediate_size
             _a    : Any     					=hidden_act
             _a    : Dict     					=hidden_dropout_prob
             _a    : int     					=attention_probs_dropout_prob
             _a    : str     					=initializer_range
             _a    : Optional[int]     					=type_vocab_size
             _a    : Any     					=layer_norm_eps
             _a    : Any     					=use_cache
             _a    : Optional[int]     					=enable_pronunciation
             _a    : Optional[Any]     					=enable_shape
             _a    : Optional[Any]     					=pronunciation_embed_dim
             _a    : Tuple     					=pronunciation_vocab_size
             _a    : Union[str, Any]     					=shape_embed_dim
             _a    : Any     					=shape_vocab_size
             _a    : Tuple     					=concat_input
             _a    : List[str]     					=position_embedding_type
             _a    : List[str]     					=classifier_dropout
             super().__init__(pad_token_id=SCREAMING_SNAKE_CASE     ,						**SCREAMING_SNAKE_CASE	)


