"""simple docstring"""




import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
			from transformers import LlamaTokenizerFast
except ImportError as e:
			warnings.warn(e)
			warnings.warn(
			    "The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"
			)
			__lowerCAmelCase     :				Union[str, Any]    				=    None
__lowerCAmelCase     :				Optional[int]    				=    {
    "7B": 1_10_08,
    "13B": 1_38_24,
    "30B": 1_79_20,
    "65B": 2_20_16,
    "70B": 2_86_72,
}
__lowerCAmelCase     :				Dict    				=    {
    "7B": 1,
    "7Bf": 1,
    "13B": 2,
    "13Bf": 2,
    "30B": 4,
    "65B": 8,
    "70B": 8,
    "70Bf": 8,
}







def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__=1      ,	lowerCamelCase__=256  ):





		"""simple docstring"""




		return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3  )  ) + multiple_of - 1) // multiple_of)







def 				_UpperCAmelCase		(							lowerCamelCase__  ):





		"""simple docstring"""




		with open(lowerCamelCase__      ,	"""r"""  ) as f:
				return json.load(lowerCamelCase__  )







def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__  ):





		"""simple docstring"""




		with open(lowerCamelCase__      ,	"""w"""  ) as f:
				json.dump(lowerCamelCase__      ,	lowerCamelCase__  )







def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__=True  ):





		"""simple docstring"""




		os.makedirs(lowerCamelCase__      ,	exist_ok=lowerCamelCase__  )
		lowerCAmelCase__   = os.path.join(lowerCamelCase__      ,	"""tmp"""  )
		os.makedirs(lowerCamelCase__      ,	exist_ok=lowerCamelCase__  )

		lowerCAmelCase__   = read_json(os.path.join(lowerCamelCase__      ,	"""params.json"""  )  )
		lowerCAmelCase__   = NUM_SHARDS[model_size]
		lowerCAmelCase__   = params["""n_layers"""]
		lowerCAmelCase__   = params["""n_heads"""]
		lowerCAmelCase__   = n_heads // num_shards
		lowerCAmelCase__   = params["""dim"""]
		lowerCAmelCase__   = dim // n_heads
		lowerCAmelCase__   = 1_00_00.0
		lowerCAmelCase__   = 1.0 / (base ** (torch.arange(0      ,	lowerCamelCase__      ,	2  ).float() / dims_per_head))

		if "n_kv_heads" in params:
				lowerCAmelCase__   = params["""n_kv_heads"""]  # for GQA / MQA
				lowerCAmelCase__   = n_heads_per_shard // num_key_value_heads
				lowerCAmelCase__   = dim // num_key_value_heads
		else:  # compatibility with other checkpoints
				lowerCAmelCase__   = n_heads
				lowerCAmelCase__   = n_heads_per_shard
				lowerCAmelCase__   = dim

		# permute for sliced rotary
		def permute(lowerCamelCase__      ,	lowerCamelCase__=n_heads      ,	lowerCamelCase__=dim      ,	lowerCamelCase__=dim  ):
				return w.view(lowerCamelCase__      ,	dima // n_heads // 2      ,	2      ,	lowerCamelCase__  ).transpose(1      ,	2  ).reshape(lowerCamelCase__      ,	lowerCamelCase__  )

		print(f"""Fetching all parameters from the checkpoint at {input_base_path}."""  )
		# Load weights
		if model_size == "7B":
				# Not sharded
				# (The sharded implementation would also work, but this is simpler.)
				lowerCAmelCase__   = torch.load(os.path.join(lowerCamelCase__      ,	"""consolidated.00.pth"""  )      ,	map_location="""cpu"""  )
		else:
				# Sharded
				lowerCAmelCase__   = [
				    torch.load(os.path.join(lowerCamelCase__      ,	f"""consolidated.{i:02d}.pth"""  )      ,	map_location="""cpu"""  )
				    for i in range(lowerCamelCase__  )
				]
		lowerCAmelCase__   = 0
		lowerCAmelCase__   = {"""weight_map""": {}}
		for layer_i in range(lowerCamelCase__  ):
				lowerCAmelCase__   = f"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
				if model_size == "7B":
						# Unsharded
						lowerCAmelCase__   = {
						    f"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
						        loaded[f"""layers.{layer_i}.attention.wq.weight"""]  ),
						    f"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
						        loaded[f"""layers.{layer_i}.attention.wk.weight"""]  ),
						    f"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[f"""layers.{layer_i}.attention.wv.weight"""],
						    f"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[f"""layers.{layer_i}.attention.wo.weight"""],
						    f"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w1.weight"""],
						    f"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w2.weight"""],
						    f"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w3.weight"""],
						    f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[f"""layers.{layer_i}.attention_norm.weight"""],
						    f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[f"""layers.{layer_i}.ffn_norm.weight"""],
						}
				else:
						# Sharded
						# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
						# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
						# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

						lowerCAmelCase__   = {
						    f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
						        f"""layers.{layer_i}.attention_norm.weight"""
						    ].clone(),
						    f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
						        f"""layers.{layer_i}.ffn_norm.weight"""
						    ].clone(),
						}
						lowerCAmelCase__   = permute(
						    torch.cat(
						        [
						            loaded[i][f"""layers.{layer_i}.attention.wq.weight"""].view(lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__  )
						            for i in range(lowerCamelCase__  )
						        ]      ,	dim=0      ,	).reshape(lowerCamelCase__      ,	lowerCamelCase__  )  )
						lowerCAmelCase__   = permute(
						    torch.cat(
						        [
						            loaded[i][f"""layers.{layer_i}.attention.wk.weight"""].view(
						                lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__  )
						            for i in range(lowerCamelCase__  )
						        ]      ,	dim=0      ,	).reshape(lowerCamelCase__      ,	lowerCamelCase__  )      ,	lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__      ,	)
						lowerCAmelCase__   = torch.cat(
						    [
						        loaded[i][f"""layers.{layer_i}.attention.wv.weight"""].view(
						            lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__  )
						        for i in range(lowerCamelCase__  )
						    ]      ,	dim=0      ,	).reshape(lowerCamelCase__      ,	lowerCamelCase__  )

						lowerCAmelCase__   = torch.cat(
						    [loaded[i][f"""layers.{layer_i}.attention.wo.weight"""] for i in range(lowerCamelCase__  )]      ,	dim=1  )
						lowerCAmelCase__   = torch.cat(
						    [loaded[i][f"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(lowerCamelCase__  )]      ,	dim=0  )
						lowerCAmelCase__   = torch.cat(
						    [loaded[i][f"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(lowerCamelCase__  )]      ,	dim=1  )
						lowerCAmelCase__   = torch.cat(
						    [loaded[i][f"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(lowerCamelCase__  )]      ,	dim=0  )

				lowerCAmelCase__   = inv_freq
				for k, v in state_dict.items():
						lowerCAmelCase__   = filename
						param_count += v.numel()
				torch.save(lowerCamelCase__      ,	os.path.join(lowerCamelCase__      ,	lowerCamelCase__  )  )

		lowerCAmelCase__   = f"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
		if model_size == "7B":
				# Unsharded
				lowerCAmelCase__   = {
				    """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
				    """model.norm.weight""": loaded["""norm.weight"""],
				    """lm_head.weight""": loaded["""output.weight"""],
				}
		else:
				lowerCAmelCase__   = {
				    """model.norm.weight""": loaded[0]["""norm.weight"""],
				    """model.embed_tokens.weight""": torch.cat(
				        [loaded[i]["""tok_embeddings.weight"""] for i in range(lowerCamelCase__  )]      ,	dim=1  ),
				    """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(lowerCamelCase__  )]      ,	dim=0  ),
				}

		for k, v in state_dict.items():
				lowerCAmelCase__   = filename
				param_count += v.numel()
		torch.save(lowerCamelCase__      ,	os.path.join(lowerCamelCase__      ,	lowerCamelCase__  )  )

		# Write configs
		lowerCAmelCase__   = {"""total_size""": param_count * 2}
		write_json(lowerCamelCase__      ,	os.path.join(lowerCamelCase__      ,	"""pytorch_model.bin.index.json"""  )  )
		lowerCAmelCase__   = params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
		lowerCAmelCase__   = params["""multiple_of"""] if """multiple_of""" in params else 256
		lowerCAmelCase__   = LlamaConfig(
		    hidden_size=lowerCamelCase__      ,	intermediate_size=compute_intermediate_size(lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__  )      ,	num_attention_heads=params["""n_heads"""]      ,	num_hidden_layers=params["""n_layers"""]      ,	rms_norm_eps=params["""norm_eps"""]      ,	num_key_value_heads=lowerCamelCase__      ,	)
		config.save_pretrained(lowerCamelCase__  )

		# Make space so we can load the model properly now.
		del state_dict
		del loaded
		gc.collect()

		print("""Loading the checkpoint in a Llama model."""  )
		lowerCAmelCase__   = LlamaForCausalLM.from_pretrained(lowerCamelCase__      ,	torch_dtype=torch.floataa      ,	low_cpu_mem_usage=lowerCamelCase__  )
		# Avoid saving this as part of the config.
		del model.config._name_or_path

		print("""Saving in the Transformers format."""  )
		model.save_pretrained(lowerCamelCase__      ,	safe_serialization=lowerCamelCase__  )
		shutil.rmtree(lowerCamelCase__  )







def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__  ):





		"""simple docstring"""




		lowerCAmelCase__   = LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
		print(f"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""  )
		lowerCAmelCase__   = tokenizer_class(lowerCamelCase__  )
		tokenizer.save_pretrained(lowerCamelCase__  )







def 				_UpperCAmelCase		(							):





		"""simple docstring"""




		lowerCAmelCase__   = argparse.ArgumentParser()
		parser.add_argument(
		    """--input_dir"""      ,	help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""      ,	)
		parser.add_argument(
		    """--model_size"""      ,	choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]      ,	)
		parser.add_argument(
		    """--output_dir"""      ,	help="""Location to write HF model and tokenizer"""      ,	)
		parser.add_argument("""--safe_serialization"""      ,	type=lowerCamelCase__      ,	help="""Whether or not to save using `safetensors`."""  )
		lowerCAmelCase__   = parser.parse_args()
		if args.model_size != "tokenizer_only":
				write_model(
				    model_path=args.output_dir      ,	input_base_path=os.path.join(args.input_dir      ,	args.model_size  )      ,	model_size=args.model_size      ,	safe_serialization=args.safe_serialization      ,	)
		lowerCAmelCase__   = os.path.join(args.input_dir      ,	"""tokenizer.model"""  )
		write_tokenizer(args.output_dir      ,	lowerCamelCase__  )


if __name__ == "__main__":
			main()





"""simple docstring"""




from typing import Optional, Tuple

import jax
import jax.numpy as jnp
from flax import linen as nn
from flax.core.frozen_dict import FrozenDict
from transformers import CLIPConfig, FlaxPreTrainedModel
from transformers.models.clip.modeling_flax_clip import FlaxCLIPVisionModule







def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__=1e-12  ):





		"""simple docstring"""




		lowerCAmelCase__   = jnp.divide(emb_a.T      ,	jnp.clip(jnp.linalg.norm(lowerCamelCase__      ,	axis=1  )      ,	a_min=lowerCamelCase__  )  ).T
		lowerCAmelCase__   = jnp.divide(emb_a.T      ,	jnp.clip(jnp.linalg.norm(lowerCamelCase__      ,	axis=1  )      ,	a_min=lowerCamelCase__  )  ).T
		return jnp.matmul(lowerCamelCase__      ,	norm_emb_a.T  )

class    a_   (  nn.Module      ):
					UpperCamelCase_			:						CLIPConfig
					UpperCamelCase_			:						jnp.dtype      =     jnp.floataa




					def     _SCREAMING_SNAKE_CASE    (						self    :  Tuple  ):
							lowerCAmelCase__   = FlaxCLIPVisionModule(self.config.vision_config  )
							lowerCAmelCase__   = nn.Dense(self.config.projection_dim	,    use_bias=snake_case__	,    dtype=self.dtype  )

							lowerCAmelCase__   = self.param("""concept_embeds"""	,    jax.nn.initializers.ones	,    (17, self.config.projection_dim)  )
							lowerCAmelCase__   = self.param(
							    """special_care_embeds"""	,    jax.nn.initializers.ones	,    (3, self.config.projection_dim)  )

							lowerCAmelCase__   = self.param("""concept_embeds_weights"""	,    jax.nn.initializers.ones	,    (17,)  )
							lowerCAmelCase__   = self.param("""special_care_embeds_weights"""	,    jax.nn.initializers.ones	,    (3,)  )




					def __call__(						self    :  Optional[int]	,    snake_case__    :  int  ):
							lowerCAmelCase__   = self.vision_model(snake_case__  )[1]
							lowerCAmelCase__   = self.visual_projection(snake_case__  )

							lowerCAmelCase__   = jax_cosine_distance(snake_case__	,    self.special_care_embeds  )
							lowerCAmelCase__   = jax_cosine_distance(snake_case__	,    self.concept_embeds  )

							# increase this value to create a stronger `nfsw` filter
							# at the cost of increasing the possibility of filtering benign image inputs
							lowerCAmelCase__   = 0.0

							lowerCAmelCase__   = special_cos_dist - self.special_care_embeds_weights[None, :] + adjustment
							lowerCAmelCase__   = jnp.round(snake_case__	,    3  )
							lowerCAmelCase__   = jnp.any(special_scores > 0	,    axis=1	,    keepdims=snake_case__  )
							# Use a lower threshold if an image has any special care concept
							lowerCAmelCase__   = is_special_care * 0.01

							lowerCAmelCase__   = cos_dist - self.concept_embeds_weights[None, :] + special_adjustment
							lowerCAmelCase__   = jnp.round(snake_case__	,    3  )
							lowerCAmelCase__   = jnp.any(concept_scores > 0	,    axis=1  )

							return has_nsfw_concepts

class    a_   (  __UpperCamelCase      ):
					UpperCamelCase_			:						Any      =     CLIPConfig
					UpperCamelCase_			:						str      =     "clip_input"
					UpperCamelCase_			:						str      =     FlaxStableDiffusionSafetyCheckerModule




					def __init__(						self    :  Optional[int]	,    snake_case__    :  CLIPConfig	,    snake_case__    :  Optional[Tuple] = None	,    snake_case__    :  int = 0	,    snake_case__    :  jnp.dtype = jnp.floataa	,    snake_case__    :  bool = True	,    **snake_case__    :  str	,    ):
							if input_shape is None:
									lowerCAmelCase__   = (1, 224, 224, 3)
							lowerCAmelCase__   = self.module_class(config=snake_case__	,    dtype=snake_case__	,    **snake_case__  )
							super().__init__(snake_case__	,    snake_case__	,    input_shape=snake_case__	,    seed=snake_case__	,    dtype=snake_case__	,    _do_init=_do_init  )




					def     _SCREAMING_SNAKE_CASE    (						self    :  Tuple	,    snake_case__    :  jax.random.KeyArray	,    snake_case__    :  Tuple	,    snake_case__    :  FrozenDict = None  ):
							# init input tensor
							lowerCAmelCase__   = jax.random.normal(snake_case__	,    snake_case__  )

							lowerCAmelCase__ ,       lowerCAmelCase__   = jax.random.split(snake_case__  )
							lowerCAmelCase__   = {"""params""": params_rng, """dropout""": dropout_rng}

							lowerCAmelCase__   = self.module.init(snake_case__	,    snake_case__  )["""params"""]

							return random_params




					def __call__(						self    :  Union[str, Any]	,    snake_case__    :  Union[str, Any]	,    snake_case__    :  dict = None	,    ):
							lowerCAmelCase__   = jnp.transpose(snake_case__	,    (0, 2, 3, 1)  )

							return self.module.apply(
							    {"""params""": params or self.params}	,    jnp.array(snake_case__	,    dtype=jnp.floataa  )	,    rngs={}	,    )




