

"""simple docstring"""




import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
						from transformers import LlamaTokenizerFast
except ImportError as e:
						warnings.warn(e)
						warnings.warn(
						    "The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"
						)
						A_							:	str      =       None
A_							:	Union[str, Any]      =       {
    '7B': 11_008,
    '13B': 13_824,
    '30B': 17_920,
    '65B': 22_016,
    '70B': 28_672,
}
A_							:	Any      =       {
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}

def 		A				(   snake_case__						, snake_case__=1						, snake_case__=2_56			):




			'''simple docstring'''


			return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3			)			) + multiple_of - 1) // multiple_of)

def 		A				(   snake_case__			):




			'''simple docstring'''


			with open(snake_case__						, """r"""			) as f:
						return json.load(snake_case__			)

def 		A				(   snake_case__						, snake_case__			):




			'''simple docstring'''


			with open(snake_case__						, """w"""			) as f:
						json.dump(snake_case__						, snake_case__			)

def 		A				(   snake_case__						, snake_case__						, snake_case__						, snake_case__=True			):




			'''simple docstring'''


			os.makedirs(snake_case__						, exist_ok=snake_case__			)
			SCREAMING_SNAKE_CASE__									=					os.path.join(snake_case__						, """tmp"""			)
			os.makedirs(snake_case__						, exist_ok=snake_case__			)

			SCREAMING_SNAKE_CASE__									=					read_json(os.path.join(snake_case__						, """params.json"""			)			)
			SCREAMING_SNAKE_CASE__									=					NUM_SHARDS[model_size]
			SCREAMING_SNAKE_CASE__									=					params["""n_layers"""]
			SCREAMING_SNAKE_CASE__									=					params["""n_heads"""]
			SCREAMING_SNAKE_CASE__									=					n_heads // num_shards
			SCREAMING_SNAKE_CASE__									=					params["""dim"""]
			SCREAMING_SNAKE_CASE__									=					dim // n_heads
			SCREAMING_SNAKE_CASE__									=					1_00_00.0
			SCREAMING_SNAKE_CASE__									=					1.0 / (base ** (torch.arange(0						, snake_case__						, 2			).float() / dims_per_head))

			if "n_kv_heads" in params:
						SCREAMING_SNAKE_CASE__									=					params["""n_kv_heads"""]  # for GQA / MQA
						SCREAMING_SNAKE_CASE__									=					n_heads_per_shard // num_key_value_heads
						SCREAMING_SNAKE_CASE__									=					dim // num_key_value_heads
			else:  # compatibility with other checkpoints
						SCREAMING_SNAKE_CASE__									=					n_heads
						SCREAMING_SNAKE_CASE__									=					n_heads_per_shard
						SCREAMING_SNAKE_CASE__									=					dim

			# permute for sliced rotary
			def permute(snake_case__						, snake_case__=n_heads						, snake_case__=dim						, snake_case__=dim			):
						return w.view(snake_case__						, dima // n_heads // 2						, 2						, snake_case__			).transpose(1						, 2			).reshape(snake_case__						, snake_case__			)

			print(f"""Fetching all parameters from the checkpoint at {input_base_path}."""			)
			# Load weights
			if model_size == "7B":
						# Not sharded
						# (The sharded implementation would also work, but this is simpler.)
						SCREAMING_SNAKE_CASE__									=					torch.load(os.path.join(snake_case__						, """consolidated.00.pth"""			)						, map_location="""cpu"""			)
			else:
						# Sharded
						SCREAMING_SNAKE_CASE__									=					[
						    torch.load(os.path.join(snake_case__						, f"""consolidated.{i:02d}.pth"""			)						, map_location="""cpu"""			)
						    for i in range(snake_case__			)
						]
			SCREAMING_SNAKE_CASE__									=					0
			SCREAMING_SNAKE_CASE__									=					{"""weight_map""": {}}
			for layer_i in range(snake_case__			):
						SCREAMING_SNAKE_CASE__									=					f"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
						if model_size == "7B":
									# Unsharded
									SCREAMING_SNAKE_CASE__									=					{
									    f"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
									        loaded[f"""layers.{layer_i}.attention.wq.weight"""]			),
									    f"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
									        loaded[f"""layers.{layer_i}.attention.wk.weight"""]			),
									    f"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[f"""layers.{layer_i}.attention.wv.weight"""],
									    f"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[f"""layers.{layer_i}.attention.wo.weight"""],
									    f"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w1.weight"""],
									    f"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w2.weight"""],
									    f"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w3.weight"""],
									    f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[f"""layers.{layer_i}.attention_norm.weight"""],
									    f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[f"""layers.{layer_i}.ffn_norm.weight"""],
									}
						else:
									# Sharded
									# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
									# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
									# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

									SCREAMING_SNAKE_CASE__									=					{
									    f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
									        f"""layers.{layer_i}.attention_norm.weight"""
									    ].clone(),
									    f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
									        f"""layers.{layer_i}.ffn_norm.weight"""
									    ].clone(),
									}
									SCREAMING_SNAKE_CASE__									=					permute(
									    torch.cat(
									        [
									            loaded[i][f"""layers.{layer_i}.attention.wq.weight"""].view(snake_case__						, snake_case__						, snake_case__			)
									            for i in range(snake_case__			)
									        ]						, dim=0						, ).reshape(snake_case__						, snake_case__			)			)
									SCREAMING_SNAKE_CASE__									=					permute(
									    torch.cat(
									        [
									            loaded[i][f"""layers.{layer_i}.attention.wk.weight"""].view(
									                snake_case__						, snake_case__						, snake_case__			)
									            for i in range(snake_case__			)
									        ]						, dim=0						, ).reshape(snake_case__						, snake_case__			)						, snake_case__						, snake_case__						, snake_case__						, )
									SCREAMING_SNAKE_CASE__									=					torch.cat(
									    [
									        loaded[i][f"""layers.{layer_i}.attention.wv.weight"""].view(
									            snake_case__						, snake_case__						, snake_case__			)
									        for i in range(snake_case__			)
									    ]						, dim=0						, ).reshape(snake_case__						, snake_case__			)

									SCREAMING_SNAKE_CASE__									=					torch.cat(
									    [loaded[i][f"""layers.{layer_i}.attention.wo.weight"""] for i in range(snake_case__			)]						, dim=1			)
									SCREAMING_SNAKE_CASE__									=					torch.cat(
									    [loaded[i][f"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(snake_case__			)]						, dim=0			)
									SCREAMING_SNAKE_CASE__									=					torch.cat(
									    [loaded[i][f"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(snake_case__			)]						, dim=1			)
									SCREAMING_SNAKE_CASE__									=					torch.cat(
									    [loaded[i][f"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(snake_case__			)]						, dim=0			)

						SCREAMING_SNAKE_CASE__									=					inv_freq
						for k, v in state_dict.items():
									SCREAMING_SNAKE_CASE__									=					filename
									param_count += v.numel()
						torch.save(snake_case__						, os.path.join(snake_case__						, snake_case__			)			)

			SCREAMING_SNAKE_CASE__									=					f"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
			if model_size == "7B":
						# Unsharded
						SCREAMING_SNAKE_CASE__									=					{
						    """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
						    """model.norm.weight""": loaded["""norm.weight"""],
						    """lm_head.weight""": loaded["""output.weight"""],
						}
			else:
						SCREAMING_SNAKE_CASE__									=					{
						    """model.norm.weight""": loaded[0]["""norm.weight"""],
						    """model.embed_tokens.weight""": torch.cat(
						        [loaded[i]["""tok_embeddings.weight"""] for i in range(snake_case__			)]						, dim=1			),
						    """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(snake_case__			)]						, dim=0			),
						}

			for k, v in state_dict.items():
						SCREAMING_SNAKE_CASE__									=					filename
						param_count += v.numel()
			torch.save(snake_case__						, os.path.join(snake_case__						, snake_case__			)			)

			# Write configs
			SCREAMING_SNAKE_CASE__									=					{"""total_size""": param_count * 2}
			write_json(snake_case__						, os.path.join(snake_case__						, """pytorch_model.bin.index.json"""			)			)
			SCREAMING_SNAKE_CASE__									=					params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
			SCREAMING_SNAKE_CASE__									=					params["""multiple_of"""] if """multiple_of""" in params else 2_56
			SCREAMING_SNAKE_CASE__									=					LlamaConfig(
			    hidden_size=snake_case__						, intermediate_size=compute_intermediate_size(snake_case__						, snake_case__						, snake_case__			)						, num_attention_heads=params["""n_heads"""]						, num_hidden_layers=params["""n_layers"""]						, rms_norm_eps=params["""norm_eps"""]						, num_key_value_heads=snake_case__						, )
			config.save_pretrained(snake_case__			)

			# Make space so we can load the model properly now.
			del state_dict
			del loaded
			gc.collect()

			print("""Loading the checkpoint in a Llama model."""			)
			SCREAMING_SNAKE_CASE__									=					LlamaForCausalLM.from_pretrained(snake_case__						, torch_dtype=torch.floataa						, low_cpu_mem_usage=snake_case__			)
			# Avoid saving this as part of the config.
			del model.config._name_or_path

			print("""Saving in the Transformers format."""			)
			model.save_pretrained(snake_case__						, safe_serialization=snake_case__			)
			shutil.rmtree(snake_case__			)

def 		A				(   snake_case__						, snake_case__			):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__									=					LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
			print(f"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""			)
			SCREAMING_SNAKE_CASE__									=					tokenizer_class(snake_case__			)
			tokenizer.save_pretrained(snake_case__			)

def 		A				(   ):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__									=					argparse.ArgumentParser()
			parser.add_argument(
			    """--input_dir"""						, help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""						, )
			parser.add_argument(
			    """--model_size"""						, choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]						, )
			parser.add_argument(
			    """--output_dir"""						, help="""Location to write HF model and tokenizer"""						, )
			parser.add_argument("""--safe_serialization"""						, type=snake_case__						, help="""Whether or not to save using `safetensors`."""			)
			SCREAMING_SNAKE_CASE__									=					parser.parse_args()
			if args.model_size != "tokenizer_only":
						write_model(
						    model_path=args.output_dir						, input_base_path=os.path.join(args.input_dir						, args.model_size			)						, model_size=args.model_size						, safe_serialization=args.safe_serialization						, )
			SCREAMING_SNAKE_CASE__									=					os.path.join(args.input_dir						, """tokenizer.model"""			)
			write_tokenizer(args.output_dir						, snake_case__			)


if __name__ == "__main__":
						main()








from __future__ import annotations

from scipy.special import comb  # type: ignore




class     snake_case__							:
				"""simple docstring"""



				def __init__(      self				:     Any					,      __lowerCamelCase				:     list[tuple[float, float]]							)					->     Tuple:
							a    =			list_of_points
							# Degree determines the flexibility of the curve.
							# Degree = 1 will produce a straight line.
							a    =			len(__lowerCamelCase							) - 1





				def 							__UpperCAmelCase							(      self				:     Union[str, Any]					,      __lowerCamelCase				:     float							)					->     list[float]:
							assert 0 <= t <= 1, "Time t must be between 0 and 1."
							a    =			[]
							for i in range(len(self.list_of_points							)							):
										# basis function for each i
										output_values.append(
										    comb(self.degree					,      __lowerCamelCase							) * ((1 - t) ** (self.degree - i)) * (t**i)							)
							# the basis must sum up to 1 for it to produce a valid Bezier curve.
							assert round(sum(__lowerCamelCase							)					,      5							) == 1
							return output_values





				def 							__UpperCAmelCase							(      self				:     Optional[Any]					,      __lowerCamelCase				:     float							)					->     tuple[float, float]:
							assert 0 <= t <= 1, "Time t must be between 0 and 1."

							a    =			self.basis_function(__lowerCamelCase							)
							a    =			0.0
							a    =			0.0
							for i in range(len(self.list_of_points							)							):
										# For all points, sum up the product of i-th basis function and i-th point.
										x += basis_function[i] * self.list_of_points[i][0]
										y += basis_function[i] * self.list_of_points[i][1]
							return (x, y)





				def 							__UpperCAmelCase							(      self				:     Optional[Any]					,      __lowerCamelCase				:     float = 0.01							)					->     List[str]:
							from matplotlib import pyplot as plt  # type: ignore

							a    =			[]  # x coordinates of points to plot
							a    =			[]  # y coordinates of points to plot

							a    =			0.0
							while t <= 1:
										a    =			self.bezier_curve_function(__lowerCamelCase							)
										to_plot_x.append(value[0]							)
										to_plot_y.append(value[1]							)
										t += step_size

							a    =			[i[0] for i in self.list_of_points]
							a    =			[i[1] for i in self.list_of_points]

							plt.plot(
							    __lowerCamelCase					,      __lowerCamelCase					,      color="blue"					,      label="Curve of Degree " + str(self.degree							)					,      )
							plt.scatter(__lowerCamelCase					,      __lowerCamelCase					,      color="red"					,      label="Control Points"							)
							plt.legend()
							plt.show()


if __name__ == "__main__":
		import doctest

		doctest.testmod()

		BezierCurve([(1, 2), (3, 5)]).plot_curve()  # degree 1
		BezierCurve([(0, 0), (5, 5), (5, 0)]).plot_curve()  # degree 2
		BezierCurve([(0, 0), (5, 5), (5, 0), (2.5, -2.5)]).plot_curve()  # degree 3






