



import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
     from transformers import LlamaTokenizerFast
except ImportError as e:
     warnings.warn(e)
     warnings.warn(
         """The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"""
     )
     SCREAMING_SNAKE_CASE_:Any             =						None
SCREAMING_SNAKE_CASE_:List[str]             =						{
    """7B""": 11_008,
    """13B""": 13_824,
    """30B""": 17_920,
    """65B""": 22_016,
    """70B""": 28_672,
}
SCREAMING_SNAKE_CASE_:List[str]             =						{
    """7B""": 1,
    """7Bf""": 1,
    """13B""": 2,
    """13Bf""": 2,
    """30B""": 4,
    """65B""": 8,
    """70B""": 8,
    """70Bf""": 8,
}




def 	__UpperCamelCase	(  _lowerCAmelCase      ,     _lowerCAmelCase=1      ,     _lowerCAmelCase=256		)       ->  Optional[int]:



    """simple docstring"""






    return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3		)		) + multiple_of - 1) // multiple_of)




def 	__UpperCamelCase	(  _lowerCAmelCase		)       ->  Optional[Any]:



    """simple docstring"""






    with open(_lowerCAmelCase      ,     """r"""		) as f:
        return json.load(_lowerCAmelCase		)




def 	__UpperCamelCase	(  _lowerCAmelCase      ,     _lowerCAmelCase		)       ->  Optional[Any]:



    """simple docstring"""






    with open(_lowerCAmelCase      ,     """w"""		) as f:
        json.dump(_lowerCAmelCase      ,     _lowerCAmelCase		)




def 	__UpperCamelCase	(  _lowerCAmelCase      ,     _lowerCAmelCase      ,     _lowerCAmelCase      ,     _lowerCAmelCase=True		)       ->  List[Any]:



    """simple docstring"""






    os.makedirs(_lowerCAmelCase      ,     exist_ok=_lowerCAmelCase		)
    A :       Dict          =   os.path.join(_lowerCAmelCase      ,     """tmp"""		)
    os.makedirs(_lowerCAmelCase      ,     exist_ok=_lowerCAmelCase		)

    A :       int          =   read_json(os.path.join(_lowerCAmelCase      ,     """params.json"""		)		)
    A :       List[str]          =   NUM_SHARDS[model_size]
    A :       int          =   params["""n_layers"""]
    A :       Any          =   params["""n_heads"""]
    A :       Tuple          =   n_heads // num_shards
    A :       Optional[int]          =   params["""dim"""]
    A :       List[str]          =   dim // n_heads
    A :       Union[str, Any]          =   10_000.0
    A :       Union[str, Any]          =   1.0 / (base ** (torch.arange(0      ,     _lowerCAmelCase      ,     2		).float() / dims_per_head))

    if "n_kv_heads" in params:
        A :       str          =   params["""n_kv_heads"""]  # for GQA / MQA
        A :       str          =   n_heads_per_shard // num_key_value_heads
        A :       List[str]          =   dim // num_key_value_heads
    else:  # compatibility with other checkpoints
        A :       Dict          =   n_heads
        A :       Dict          =   n_heads_per_shard
        A :       Dict          =   dim

    # permute for sliced rotary
    def permute(_lowerCAmelCase      ,     _lowerCAmelCase=n_heads      ,     _lowerCAmelCase=dim      ,     _lowerCAmelCase=dim		):
        return w.view(_lowerCAmelCase      ,     dima // n_heads // 2      ,     2      ,     _lowerCAmelCase		).transpose(1      ,     2		).reshape(_lowerCAmelCase      ,     _lowerCAmelCase		)

    print(f'''Fetching all parameters from the checkpoint at {input_base_path}.'''		)
    # Load weights
    if model_size == "7B":
        # Not sharded
        # (The sharded implementation would also work, but this is simpler.)
        A :       Union[str, Any]          =   torch.load(os.path.join(_lowerCAmelCase      ,     """consolidated.00.pth"""		)      ,     map_location="""cpu"""		)
    else:
        # Sharded
        A :       Optional[Any]          =   [
            torch.load(os.path.join(_lowerCAmelCase      ,     f'''consolidated.{i:02d}.pth'''		)      ,     map_location="""cpu"""		)
            for i in range(_lowerCAmelCase		)
        ]
    A :       Optional[int]          =   0
    A :       List[Any]          =   {"""weight_map""": {}}
    for layer_i in range(_lowerCAmelCase		):
        A :       int          =   f'''pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'''
        if model_size == "7B":
            # Unsharded
            A :       Any          =   {
                f'''model.layers.{layer_i}.self_attn.q_proj.weight''': permute(
                    loaded[f'''layers.{layer_i}.attention.wq.weight''']		),
                f'''model.layers.{layer_i}.self_attn.k_proj.weight''': permute(
                    loaded[f'''layers.{layer_i}.attention.wk.weight''']		),
                f'''model.layers.{layer_i}.self_attn.v_proj.weight''': loaded[f'''layers.{layer_i}.attention.wv.weight'''],
                f'''model.layers.{layer_i}.self_attn.o_proj.weight''': loaded[f'''layers.{layer_i}.attention.wo.weight'''],
                f'''model.layers.{layer_i}.mlp.gate_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w1.weight'''],
                f'''model.layers.{layer_i}.mlp.down_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w2.weight'''],
                f'''model.layers.{layer_i}.mlp.up_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w3.weight'''],
                f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[f'''layers.{layer_i}.attention_norm.weight'''],
                f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[f'''layers.{layer_i}.ffn_norm.weight'''],
            }
        else:
            # Sharded
            # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
            # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
            # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

            A :       str          =   {
                f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[0][
                    f'''layers.{layer_i}.attention_norm.weight'''
                ].clone(),
                f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[0][
                    f'''layers.{layer_i}.ffn_norm.weight'''
                ].clone(),
            }
            A :       str          =   permute(
                torch.cat(
                    [
                        loaded[i][f'''layers.{layer_i}.attention.wq.weight'''].view(_lowerCAmelCase      ,     _lowerCAmelCase      ,     _lowerCAmelCase		)
                        for i in range(_lowerCAmelCase		)
                    ]      ,     dim=0      ,     ).reshape(_lowerCAmelCase      ,     _lowerCAmelCase		)		)
            A :       Union[str, Any]          =   permute(
                torch.cat(
                    [
                        loaded[i][f'''layers.{layer_i}.attention.wk.weight'''].view(
                            _lowerCAmelCase      ,     _lowerCAmelCase      ,     _lowerCAmelCase		)
                        for i in range(_lowerCAmelCase		)
                    ]      ,     dim=0      ,     ).reshape(_lowerCAmelCase      ,     _lowerCAmelCase		)      ,     _lowerCAmelCase      ,     _lowerCAmelCase      ,     _lowerCAmelCase      ,     )
            A :       Union[str, Any]          =   torch.cat(
                [
                    loaded[i][f'''layers.{layer_i}.attention.wv.weight'''].view(
                        _lowerCAmelCase      ,     _lowerCAmelCase      ,     _lowerCAmelCase		)
                    for i in range(_lowerCAmelCase		)
                ]      ,     dim=0      ,     ).reshape(_lowerCAmelCase      ,     _lowerCAmelCase		)

            A :       Any          =   torch.cat(
                [loaded[i][f'''layers.{layer_i}.attention.wo.weight'''] for i in range(_lowerCAmelCase		)]      ,     dim=1		)
            A :       str          =   torch.cat(
                [loaded[i][f'''layers.{layer_i}.feed_forward.w1.weight'''] for i in range(_lowerCAmelCase		)]      ,     dim=0		)
            A :       int          =   torch.cat(
                [loaded[i][f'''layers.{layer_i}.feed_forward.w2.weight'''] for i in range(_lowerCAmelCase		)]      ,     dim=1		)
            A :       Union[str, Any]          =   torch.cat(
                [loaded[i][f'''layers.{layer_i}.feed_forward.w3.weight'''] for i in range(_lowerCAmelCase		)]      ,     dim=0		)

        A :       int          =   inv_freq
        for k, v in state_dict.items():
            A :       Dict          =   filename
            param_count += v.numel()
        torch.save(_lowerCAmelCase      ,     os.path.join(_lowerCAmelCase      ,     _lowerCAmelCase		)		)

    A :       Tuple          =   f'''pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'''
    if model_size == "7B":
        # Unsharded
        A :       int          =   {
            """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
            """model.norm.weight""": loaded["""norm.weight"""],
            """lm_head.weight""": loaded["""output.weight"""],
        }
    else:
        A :       Union[str, Any]          =   {
            """model.norm.weight""": loaded[0]["""norm.weight"""],
            """model.embed_tokens.weight""": torch.cat(
                [loaded[i]["""tok_embeddings.weight"""] for i in range(_lowerCAmelCase		)]      ,     dim=1		),
            """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(_lowerCAmelCase		)]      ,     dim=0		),
        }

    for k, v in state_dict.items():
        A :       Optional[int]          =   filename
        param_count += v.numel()
    torch.save(_lowerCAmelCase      ,     os.path.join(_lowerCAmelCase      ,     _lowerCAmelCase		)		)

    # Write configs
    A :       Optional[int]          =   {"""total_size""": param_count * 2}
    write_json(_lowerCAmelCase      ,     os.path.join(_lowerCAmelCase      ,     """pytorch_model.bin.index.json"""		)		)
    A :       Dict          =   params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
    A :       List[Any]          =   params["""multiple_of"""] if """multiple_of""" in params else 256
    A :       int          =   LlamaConfig(
        hidden_size=_lowerCAmelCase      ,     intermediate_size=compute_intermediate_size(_lowerCAmelCase      ,     _lowerCAmelCase      ,     _lowerCAmelCase		)      ,     num_attention_heads=params["""n_heads"""]      ,     num_hidden_layers=params["""n_layers"""]      ,     rms_norm_eps=params["""norm_eps"""]      ,     num_key_value_heads=_lowerCAmelCase      ,     )
    config.save_pretrained(_lowerCAmelCase		)

    # Make space so we can load the model properly now.
    del state_dict
    del loaded
    gc.collect()

    print("""Loading the checkpoint in a Llama model."""		)
    A :       Optional[Any]          =   LlamaForCausalLM.from_pretrained(_lowerCAmelCase      ,     torch_dtype=torch.floataa      ,     low_cpu_mem_usage=_lowerCAmelCase		)
    # Avoid saving this as part of the config.
    del model.config._name_or_path

    print("""Saving in the Transformers format."""		)
    model.save_pretrained(_lowerCAmelCase      ,     safe_serialization=_lowerCAmelCase		)
    shutil.rmtree(_lowerCAmelCase		)




def 	__UpperCamelCase	(  _lowerCAmelCase      ,     _lowerCAmelCase		)       ->  Tuple:



    """simple docstring"""






    A :       str          =   LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
    print(f'''Saving a {tokenizer_class.__name__} to {tokenizer_path}.'''		)
    A :       str          =   tokenizer_class(_lowerCAmelCase		)
    tokenizer.save_pretrained(_lowerCAmelCase		)




def 	__UpperCamelCase	(  )       ->  Any:



    """simple docstring"""






    A :       int          =   argparse.ArgumentParser()
    parser.add_argument(
        """--input_dir"""      ,     help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""      ,     )
    parser.add_argument(
        """--model_size"""      ,     choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]      ,     )
    parser.add_argument(
        """--output_dir"""      ,     help="""Location to write HF model and tokenizer"""      ,     )
    parser.add_argument("""--safe_serialization"""      ,     type=_lowerCAmelCase      ,     help="""Whether or not to save using `safetensors`."""		)
    A :       Any          =   parser.parse_args()
    if args.model_size != "tokenizer_only":
        write_model(
            model_path=args.output_dir      ,     input_base_path=os.path.join(args.input_dir      ,     args.model_size		)      ,     model_size=args.model_size      ,     safe_serialization=args.safe_serialization      ,     )
    A :       Optional[Any]          =   os.path.join(args.input_dir      ,     """tokenizer.model"""		)
    write_tokenizer(args.output_dir      ,     _lowerCAmelCase		)


if __name__ == "__main__":
     main()







def 	__UpperCamelCase	(  _lowerCAmelCase      ,     _lowerCAmelCase		)       ->  int:



    """simple docstring"""






    while second != 0:
        A :       int          =   first & second
        first ^= second
        A :       Tuple          =   c << 1
    return first


if __name__ == "__main__":
     import doctest

     doctest.testmod()

     SCREAMING_SNAKE_CASE_:int             =						int(input("""Enter the first number: """).strip())
     SCREAMING_SNAKE_CASE_:Optional[int]             =						int(input("""Enter the second number: """).strip())
     print(F"""{add(first, second) = }""")


