




'''simple docstring'''







import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
   from transformers import LlamaTokenizerFast
except ImportError as e:
   warnings.warn(e)
   warnings.warn(
       'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
   )
   A     =None
A     ={
    """7B""": 1_10_08,
    """13B""": 1_38_24,
    """30B""": 1_79_20,
    """65B""": 2_20_16,
    """70B""": 2_86_72,
}
A     ={
    """7B""": 1,
    """7Bf""": 1,
    """13B""": 2,
    """13Bf""": 2,
    """30B""": 4,
    """65B""": 8,
    """70B""": 8,
    """70Bf""": 8,
}





def 						snake_case_							(_a	:							Optional[Any]    ,   _a	:							List[Any]=1    ,   _a	:							Any=2_5_6     ):
    return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3     )     ) + multiple_of - 1) // multiple_of)





def 						snake_case_							(_a	:							Any     ):






    with open(_lowerCAmelCase    ,   '''r'''     ) as f:
        return json.load(_lowerCAmelCase     )





def 						snake_case_							(_a	:							str    ,   _a	:							List[Any]     ):






    with open(_lowerCAmelCase    ,   '''w'''     ) as f:
        json.dump(_lowerCAmelCase    ,   _lowerCAmelCase     )





def 						snake_case_							(_a	:							int    ,   _a	:							List[str]    ,   _a	:							str    ,   _a	:							Any=True     ):
    os.makedirs(_lowerCAmelCase    ,   exist_ok=_lowerCAmelCase     )
    UpperCAmelCase			       =		os.path.join(_lowerCAmelCase    ,   '''tmp'''     )
    os.makedirs(_lowerCAmelCase    ,   exist_ok=_lowerCAmelCase     )

    UpperCAmelCase			       =		read_json(os.path.join(_lowerCAmelCase    ,   '''params.json'''     )     )
    UpperCAmelCase			       =		NUM_SHARDS[model_size]
    UpperCAmelCase			       =		params["""n_layers"""]
    UpperCAmelCase			       =		params["""n_heads"""]
    UpperCAmelCase			       =		n_heads // num_shards
    UpperCAmelCase			       =		params["""dim"""]
    UpperCAmelCase			       =		dim // n_heads
    UpperCAmelCase			       =		1_0_0_0_0.0
    UpperCAmelCase			       =		1.0 / (base ** (torch.arange(0    ,   _lowerCAmelCase    ,   2     ).float() / dims_per_head))

    if "n_kv_heads" in params:
        UpperCAmelCase			       =		params["""n_kv_heads"""]  # for GQA / MQA
        UpperCAmelCase			       =		n_heads_per_shard // num_key_value_heads
        UpperCAmelCase			       =		dim // num_key_value_heads
    else:  # compatibility with other checkpoints
        UpperCAmelCase			       =		n_heads
        UpperCAmelCase			       =		n_heads_per_shard
        UpperCAmelCase			       =		dim

    # permute for sliced rotary
    def permute(_a	:							List[Any]    ,   _a	:							str=n_heads    ,   _a	:							List[str]=dim    ,   _a	:							Tuple=dim     ):
        return w.view(_lowerCAmelCase    ,   dima // n_heads // 2    ,   2    ,   _lowerCAmelCase     ).transpose(1    ,   2     ).reshape(_lowerCAmelCase    ,   _lowerCAmelCase     )

    print(F"Fetching all parameters from the checkpoint at {input_base_path}."     )
    # Load weights
    if model_size == "7B":
        # Not sharded
        # (The sharded implementation would also work, but this is simpler.)
        UpperCAmelCase			       =		torch.load(os.path.join(_lowerCAmelCase    ,   '''consolidated.00.pth'''     )    ,   map_location='''cpu'''     )
    else:
        # Sharded
        UpperCAmelCase			       =		[
            torch.load(os.path.join(_lowerCAmelCase    ,   F"consolidated.{i:02d}.pth"     )    ,   map_location='''cpu'''     )
            for i in range(_lowerCAmelCase     )
        ]
    UpperCAmelCase			       =		0
    UpperCAmelCase			       =		{"""weight_map""": {}}
    for layer_i in range(_lowerCAmelCase     ):
        UpperCAmelCase			       =		F"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"
        if model_size == "7B":
            # Unsharded
            UpperCAmelCase			       =		{
                F"model.layers.{layer_i}.self_attn.q_proj.weight": permute(
                    loaded[F"layers.{layer_i}.attention.wq.weight"]     ),
                F"model.layers.{layer_i}.self_attn.k_proj.weight": permute(
                    loaded[F"layers.{layer_i}.attention.wk.weight"]     ),
                F"model.layers.{layer_i}.self_attn.v_proj.weight": loaded[F"layers.{layer_i}.attention.wv.weight"],
                F"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[F"layers.{layer_i}.attention.wo.weight"],
                F"model.layers.{layer_i}.mlp.gate_proj.weight": loaded[F"layers.{layer_i}.feed_forward.w1.weight"],
                F"model.layers.{layer_i}.mlp.down_proj.weight": loaded[F"layers.{layer_i}.feed_forward.w2.weight"],
                F"model.layers.{layer_i}.mlp.up_proj.weight": loaded[F"layers.{layer_i}.feed_forward.w3.weight"],
                F"model.layers.{layer_i}.input_layernorm.weight": loaded[F"layers.{layer_i}.attention_norm.weight"],
                F"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[F"layers.{layer_i}.ffn_norm.weight"],
            }
        else:
            # Sharded
            # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
            # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
            # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

            UpperCAmelCase			       =		{
                F"model.layers.{layer_i}.input_layernorm.weight": loaded[0][
                    F"layers.{layer_i}.attention_norm.weight"
                ].clone(),
                F"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][
                    F"layers.{layer_i}.ffn_norm.weight"
                ].clone(),
            }
            UpperCAmelCase			       =		permute(
                torch.cat(
                    [
                        loaded[i][F"layers.{layer_i}.attention.wq.weight"].view(_lowerCAmelCase    ,   _lowerCAmelCase    ,   _lowerCAmelCase     )
                        for i in range(_lowerCAmelCase     )
                    ]    ,   dim=0    ,   ).reshape(_lowerCAmelCase    ,   _lowerCAmelCase     )     )
            UpperCAmelCase			       =		permute(
                torch.cat(
                    [
                        loaded[i][F"layers.{layer_i}.attention.wk.weight"].view(
                            _lowerCAmelCase    ,   _lowerCAmelCase    ,   _lowerCAmelCase     )
                        for i in range(_lowerCAmelCase     )
                    ]    ,   dim=0    ,   ).reshape(_lowerCAmelCase    ,   _lowerCAmelCase     )    ,   _lowerCAmelCase    ,   _lowerCAmelCase    ,   _lowerCAmelCase    ,   )
            UpperCAmelCase			       =		torch.cat(
                [
                    loaded[i][F"layers.{layer_i}.attention.wv.weight"].view(
                        _lowerCAmelCase    ,   _lowerCAmelCase    ,   _lowerCAmelCase     )
                    for i in range(_lowerCAmelCase     )
                ]    ,   dim=0    ,   ).reshape(_lowerCAmelCase    ,   _lowerCAmelCase     )

            UpperCAmelCase			       =		torch.cat(
                [loaded[i][F"layers.{layer_i}.attention.wo.weight"] for i in range(_lowerCAmelCase     )]    ,   dim=1     )
            UpperCAmelCase			       =		torch.cat(
                [loaded[i][F"layers.{layer_i}.feed_forward.w1.weight"] for i in range(_lowerCAmelCase     )]    ,   dim=0     )
            UpperCAmelCase			       =		torch.cat(
                [loaded[i][F"layers.{layer_i}.feed_forward.w2.weight"] for i in range(_lowerCAmelCase     )]    ,   dim=1     )
            UpperCAmelCase			       =		torch.cat(
                [loaded[i][F"layers.{layer_i}.feed_forward.w3.weight"] for i in range(_lowerCAmelCase     )]    ,   dim=0     )

        UpperCAmelCase			       =		inv_freq
        for k, v in state_dict.items():
            UpperCAmelCase			       =		filename
            param_count += v.numel()
        torch.save(_lowerCAmelCase    ,   os.path.join(_lowerCAmelCase    ,   _lowerCAmelCase     )     )

    UpperCAmelCase			       =		F"pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"
    if model_size == "7B":
        # Unsharded
        UpperCAmelCase			       =		{
            """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
            """model.norm.weight""": loaded["""norm.weight"""],
            """lm_head.weight""": loaded["""output.weight"""],
        }
    else:
        UpperCAmelCase			       =		{
            """model.norm.weight""": loaded[0]["""norm.weight"""],
            """model.embed_tokens.weight""": torch.cat(
                [loaded[i]['''tok_embeddings.weight'''] for i in range(_lowerCAmelCase     )]    ,   dim=1     ),
            """lm_head.weight""": torch.cat([loaded[i]['''output.weight'''] for i in range(_lowerCAmelCase     )]    ,   dim=0     ),
        }

    for k, v in state_dict.items():
        UpperCAmelCase			       =		filename
        param_count += v.numel()
    torch.save(_lowerCAmelCase    ,   os.path.join(_lowerCAmelCase    ,   _lowerCAmelCase     )     )

    # Write configs
    UpperCAmelCase			       =		{"""total_size""": param_count * 2}
    write_json(_lowerCAmelCase    ,   os.path.join(_lowerCAmelCase    ,   '''pytorch_model.bin.index.json'''     )     )
    UpperCAmelCase			       =		params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
    UpperCAmelCase			       =		params["""multiple_of"""] if """multiple_of""" in params else 2_5_6
    UpperCAmelCase			       =		LlamaConfig(
        hidden_size=_lowerCAmelCase    ,   intermediate_size=compute_intermediate_size(_lowerCAmelCase    ,   _lowerCAmelCase    ,   _lowerCAmelCase     )    ,   num_attention_heads=params['''n_heads''']    ,   num_hidden_layers=params['''n_layers''']    ,   rms_norm_eps=params['''norm_eps''']    ,   num_key_value_heads=_lowerCAmelCase    ,   )
    config.save_pretrained(_lowerCAmelCase     )

    # Make space so we can load the model properly now.
    del state_dict
    del loaded
    gc.collect()

    print('''Loading the checkpoint in a Llama model.'''     )
    UpperCAmelCase			       =		LlamaForCausalLM.from_pretrained(_lowerCAmelCase    ,   torch_dtype=torch.floataa    ,   low_cpu_mem_usage=_lowerCAmelCase     )
    # Avoid saving this as part of the config.
    del model.config._name_or_path

    print('''Saving in the Transformers format.'''     )
    model.save_pretrained(_lowerCAmelCase    ,   safe_serialization=_lowerCAmelCase     )
    shutil.rmtree(_lowerCAmelCase     )





def 						snake_case_							(_a	:							str    ,   _a	:							Optional[Any]     ):
    UpperCAmelCase			       =		LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
    print(F"Saving a {tokenizer_class.__name__} to {tokenizer_path}."     )
    UpperCAmelCase			       =		tokenizer_class(_lowerCAmelCase     )
    tokenizer.save_pretrained(_lowerCAmelCase     )





def 						snake_case_							():
    UpperCAmelCase			       =		argparse.ArgumentParser()
    parser.add_argument(
        '''--input_dir'''    ,   help='''Location of LLaMA weights, which contains tokenizer.model and model folders'''    ,   )
    parser.add_argument(
        '''--model_size'''    ,   choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only''']    ,   )
    parser.add_argument(
        '''--output_dir'''    ,   help='''Location to write HF model and tokenizer'''    ,   )
    parser.add_argument('''--safe_serialization'''    ,   type=_lowerCAmelCase    ,   help='''Whether or not to save using `safetensors`.'''     )
    UpperCAmelCase			       =		parser.parse_args()
    if args.model_size != "tokenizer_only":
        write_model(
            model_path=args.output_dir    ,   input_base_path=os.path.join(args.input_dir    ,   args.model_size     )    ,   model_size=args.model_size    ,   safe_serialization=args.safe_serialization    ,   )
    UpperCAmelCase			       =		os.path.join(args.input_dir    ,   '''tokenizer.model'''     )
    write_tokenizer(args.output_dir    ,   _lowerCAmelCase     )


if __name__ == "__main__":
   main()









import torch

from diffusers import DDIMParallelScheduler

from .test_schedulers import SchedulerCommonTest






class SCREAMING_SNAKE_CASE__     (		SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: List[str]					=      (DDIMParallelScheduler,)
    __lowerCamelCase			: int					=      (("eta", 0.0), ("num_inference_steps", 50))






    def 			_lowerCAmelCase					(       self,		**lowerCamelCase__		):
        A :       int          =   {
            """num_train_timesteps""": 1000,
            """beta_start""": 0.0001,
            """beta_end""": 0.02,
            """beta_schedule""": """linear""",
            """clip_sample""": True,
        }

        config.update(**lowerCamelCase__		)
        return config






    def 			_lowerCAmelCase					(       self,		**lowerCamelCase__		):
        A :       Dict          =   self.scheduler_classes[0]
        A :       str          =   self.get_scheduler_config(**lowerCamelCase__		)
        A :       Tuple          =   scheduler_class(**lowerCamelCase__		)

        A     ,							A :       Tuple          =   10, 0.0

        A :       Optional[int]          =   self.dummy_model()
        A :       List[str]          =   self.dummy_sample_deter

        scheduler.set_timesteps(lowerCamelCase__		)

        for t in scheduler.timesteps:
            A :       Optional[int]          =   model(lowerCamelCase__,		lowerCamelCase__		)
            A :       str          =   scheduler.step(lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__		).prev_sample

        return sample






    def 			_lowerCAmelCase					(       self		):
        for timesteps in [100, 500, 1000]:
            self.check_over_configs(num_train_timesteps=lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self		):
        for steps_offset in [0, 1]:
            self.check_over_configs(steps_offset=lowerCamelCase__		)

        A :       Tuple          =   self.scheduler_classes[0]
        A :       Optional[Any]          =   self.get_scheduler_config(steps_offset=1		)
        A :       List[Any]          =   scheduler_class(**lowerCamelCase__		)
        scheduler.set_timesteps(5		)
        assert torch.equal(scheduler.timesteps,		torch.LongTensor([801, 601, 401, 201, 1]		)		)






    def 			_lowerCAmelCase					(       self		):
        for beta_start, beta_end in zip([0.0001, 0.001, 0.01, 0.1],		[0.002, 0.02, 0.2, 2]		):
            self.check_over_configs(beta_start=lowerCamelCase__,		beta_end=lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self		):
        for schedule in ["linear", "squaredcos_cap_v2"]:
            self.check_over_configs(beta_schedule=lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self		):
        for prediction_type in ["epsilon", "v_prediction"]:
            self.check_over_configs(prediction_type=lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self		):
        for clip_sample in [True, False]:
            self.check_over_configs(clip_sample=lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self		):
        for timestep_spacing in ["trailing", "leading"]:
            self.check_over_configs(timestep_spacing=lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self		):
        for rescale_betas_zero_snr in [True, False]:
            self.check_over_configs(rescale_betas_zero_snr=lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self		):
        self.check_over_configs(thresholding=lowerCamelCase__		)
        for threshold in [0.5, 1.0, 2.0]:
            for prediction_type in ["epsilon", "v_prediction"]:
                self.check_over_configs(
                    thresholding=lowerCamelCase__,		prediction_type=lowerCamelCase__,		sample_max_value=lowerCamelCase__,		)






    def 			_lowerCAmelCase					(       self		):
        for t in [1, 10, 49]:
            self.check_over_forward(time_step=lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self		):
        for t, num_inference_steps in zip([1, 10, 50],		[10, 50, 500]		):
            self.check_over_forward(time_step=lowerCamelCase__,		num_inference_steps=lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self		):
        for t, eta in zip([1, 10, 49],		[0.0, 0.5, 1.0]		):
            self.check_over_forward(time_step=lowerCamelCase__,		eta=lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self		):
        A :       List[str]          =   self.scheduler_classes[0]
        A :       List[str]          =   self.get_scheduler_config()
        A :       Dict          =   scheduler_class(**lowerCamelCase__		)

        assert torch.sum(torch.abs(scheduler._get_variance(0,		0		) - 0.0		)		) < 1e-5
        assert torch.sum(torch.abs(scheduler._get_variance(420,		400		) - 0.1_4771		)		) < 1e-5
        assert torch.sum(torch.abs(scheduler._get_variance(980,		960		) - 0.3_2460		)		) < 1e-5
        assert torch.sum(torch.abs(scheduler._get_variance(0,		0		) - 0.0		)		) < 1e-5
        assert torch.sum(torch.abs(scheduler._get_variance(487,		486		) - 0.0_0979		)		) < 1e-5
        assert torch.sum(torch.abs(scheduler._get_variance(999,		998		) - 0.02		)		) < 1e-5






    def 			_lowerCAmelCase					(       self		):
        A :       int          =   self.scheduler_classes[0]
        A :       Any          =   self.get_scheduler_config()
        A :       Union[str, Any]          =   scheduler_class(**lowerCamelCase__		)

        A     ,							A :       List[str]          =   10, 0.0
        scheduler.set_timesteps(lowerCamelCase__		)

        A :       Any          =   self.dummy_model()
        A :       Dict          =   self.dummy_sample_deter
        A :       Dict          =   self.dummy_sample_deter + 0.1
        A :       Tuple          =   self.dummy_sample_deter - 0.1

        A :       Tuple          =   samplea.shape[0]
        A :       List[Any]          =   torch.stack([samplea, samplea, samplea],		dim=0		)
        A :       str          =   torch.arange(lowerCamelCase__		)[0:3, None].repeat(1,		lowerCamelCase__		)

        A :       Any          =   model(samples.flatten(0,		1		),		timesteps.flatten(0,		1		)		)
        A :       Any          =   scheduler.batch_step_no_noise(lowerCamelCase__,		timesteps.flatten(0,		1		),		samples.flatten(0,		1		),		lowerCamelCase__		)

        A :       List[Any]          =   torch.sum(torch.abs(lowerCamelCase__		)		)
        A :       Optional[Any]          =   torch.mean(torch.abs(lowerCamelCase__		)		)

        assert abs(result_sum.item() - 1147.7904		) < 1e-2
        assert abs(result_mean.item() - 0.4982		) < 1e-3






    def 			_lowerCAmelCase					(       self		):
        A :       Dict          =   self.full_loop()

        A :       str          =   torch.sum(torch.abs(lowerCamelCase__		)		)
        A :       Optional[Any]          =   torch.mean(torch.abs(lowerCamelCase__		)		)

        assert abs(result_sum.item() - 172.0067		) < 1e-2
        assert abs(result_mean.item() - 0.22_3967		) < 1e-3






    def 			_lowerCAmelCase					(       self		):
        A :       str          =   self.full_loop(prediction_type="""v_prediction"""		)

        A :       Any          =   torch.sum(torch.abs(lowerCamelCase__		)		)
        A :       Optional[Any]          =   torch.mean(torch.abs(lowerCamelCase__		)		)

        assert abs(result_sum.item() - 52.5302		) < 1e-2
        assert abs(result_mean.item() - 0.0684		) < 1e-3






    def 			_lowerCAmelCase					(       self		):
        # We specify different beta, so that the first alpha is 0.99
        A :       Any          =   self.full_loop(set_alpha_to_one=lowerCamelCase__,		beta_start=0.01		)
        A :       Dict          =   torch.sum(torch.abs(lowerCamelCase__		)		)
        A :       str          =   torch.mean(torch.abs(lowerCamelCase__		)		)

        assert abs(result_sum.item() - 149.8295		) < 1e-2
        assert abs(result_mean.item() - 0.1951		) < 1e-3






    def 			_lowerCAmelCase					(       self		):
        # We specify different beta, so that the first alpha is 0.99
        A :       int          =   self.full_loop(set_alpha_to_one=lowerCamelCase__,		beta_start=0.01		)
        A :       List[Any]          =   torch.sum(torch.abs(lowerCamelCase__		)		)
        A :       List[Any]          =   torch.mean(torch.abs(lowerCamelCase__		)		)

        assert abs(result_sum.item() - 149.0784		) < 1e-2
        assert abs(result_mean.item() - 0.1941		) < 1e-3


