

'''simple docstring'''
import unittest

import torch
from torch import nn

from diffusers.models.activations import get_activation

class 						__magic_name__   (							unittest.TestCase ):







 def 						_A(       self   ):
   lowercase									=get_activation('''swish'''   )

   self.assertIsInstance(snake_case_  , nn.SiLU   )

   self.assertEqual(act(torch.tensor(-1_00  , dtype=torch.floataa   )   ).item()  , 0   )
   self.assertNotEqual(act(torch.tensor(-1  , dtype=torch.floataa   )   ).item()  , 0   )
   self.assertEqual(act(torch.tensor(0  , dtype=torch.floataa   )   ).item()  , 0   )
   self.assertEqual(act(torch.tensor(20  , dtype=torch.floataa   )   ).item()  , 20   )







 def 						_A(       self   ):
   lowercase									=get_activation('''silu'''   )

   self.assertIsInstance(snake_case_  , nn.SiLU   )

   self.assertEqual(act(torch.tensor(-1_00  , dtype=torch.floataa   )   ).item()  , 0   )
   self.assertNotEqual(act(torch.tensor(-1  , dtype=torch.floataa   )   ).item()  , 0   )
   self.assertEqual(act(torch.tensor(0  , dtype=torch.floataa   )   ).item()  , 0   )
   self.assertEqual(act(torch.tensor(20  , dtype=torch.floataa   )   ).item()  , 20   )







 def 						_A(       self   ):
   lowercase									=get_activation('''mish'''   )

   self.assertIsInstance(snake_case_  , nn.Mish   )

   self.assertEqual(act(torch.tensor(-2_00  , dtype=torch.floataa   )   ).item()  , 0   )
   self.assertNotEqual(act(torch.tensor(-1  , dtype=torch.floataa   )   ).item()  , 0   )
   self.assertEqual(act(torch.tensor(0  , dtype=torch.floataa   )   ).item()  , 0   )
   self.assertEqual(act(torch.tensor(20  , dtype=torch.floataa   )   ).item()  , 20   )







 def 						_A(       self   ):
   lowercase									=get_activation('''gelu'''   )

   self.assertIsInstance(snake_case_  , nn.GELU   )

   self.assertEqual(act(torch.tensor(-1_00  , dtype=torch.floataa   )   ).item()  , 0   )
   self.assertNotEqual(act(torch.tensor(-1  , dtype=torch.floataa   )   ).item()  , 0   )
   self.assertEqual(act(torch.tensor(0  , dtype=torch.floataa   )   ).item()  , 0   )
   self.assertEqual(act(torch.tensor(20  , dtype=torch.floataa   )   ).item()  , 20   )







"""simple docstring"""


import argparse
from pathlib import Path
from typing import Dict, OrderedDict, Tuple

import torch
from audiocraft.models import MusicGen

from transformers import (
    AutoFeatureExtractor,
    AutoTokenizer,
    EncodecModel,
    MusicgenDecoderConfig,
    MusicgenForConditionalGeneration,
    MusicgenProcessor,
    TaEncoderModel,
)
from transformers.models.musicgen.modeling_musicgen import MusicgenForCausalLM
from transformers.utils import logging


logging.set_verbosity_info()
lowerCamelCase__						= logging.get_logger(__name__)


lowerCamelCase__						= ["model.decoder.embed_positions.weights"]




def        lowercase__     (       lowercase_      )       ->		Dict:




      """simple docstring"""

      if "emb" in name:
            _UpperCamelCase   :						Optional[int]    =							name.replace("emb"		,"model.decoder.embed_tokens"      )
      if "transformer" in name:
            _UpperCamelCase   :						List[Any]    =							name.replace("transformer"		,"model.decoder"      )
      if "cross_attention" in name:
            _UpperCamelCase   :						Any    =							name.replace("cross_attention"		,"encoder_attn"      )
      if "linear1" in name:
            _UpperCamelCase   :						List[str]    =							name.replace("linear1"		,"fc1"      )
      if "linear2" in name:
            _UpperCamelCase   :						Optional[Any]    =							name.replace("linear2"		,"fc2"      )
      if "norm1" in name:
            _UpperCamelCase   :						int    =							name.replace("norm1"		,"self_attn_layer_norm"      )
      if "norm_cross" in name:
            _UpperCamelCase   :						Optional[int]    =							name.replace("norm_cross"		,"encoder_attn_layer_norm"      )
      if "norm2" in name:
            _UpperCamelCase   :						Optional[Any]    =							name.replace("norm2"		,"final_layer_norm"      )
      if "out_norm" in name:
            _UpperCamelCase   :						Tuple    =							name.replace("out_norm"		,"model.decoder.layer_norm"      )
      if "linears" in name:
            _UpperCamelCase   :						List[str]    =							name.replace("linears"		,"lm_heads"      )
      if "condition_provider.conditioners.description.output_proj" in name:
            _UpperCamelCase   :						Optional[Any]    =							name.replace("condition_provider.conditioners.description.output_proj"		,"enc_to_dec_proj"      )
      return name




def        lowercase__     (       lowercase_		,lowercase_      )       ->		Tuple[Dict, Dict]:




      """simple docstring"""

      _UpperCamelCase   :						List[Any]    =							list(state_dict.keys()      )
      _UpperCamelCase   :						int    =							{}
      for key in keys:
            _UpperCamelCase   :						List[str]    =							state_dict.pop(lowercase_      )
            _UpperCamelCase   :						List[Any]    =							rename_keys(lowercase_      )
            if "in_proj_weight" in key:
                  # split fused qkv proj
                  _UpperCamelCase   :						List[Any]    =							val[:hidden_size, :]
                  _UpperCamelCase   :						List[Any]    =							val[hidden_size : 2 * hidden_size, :]
                  _UpperCamelCase   :						Tuple    =							val[-hidden_size:, :]
            elif "enc_to_dec_proj" in key:
                  _UpperCamelCase   :						Optional[Any]    =							val
            else:
                  _UpperCamelCase   :						str    =							val
      return state_dict, enc_dec_proj_state_dict




def        lowercase__     (       lowercase_      )       ->		MusicgenDecoderConfig:




      """simple docstring"""

      if checkpoint == "small":
            # default config values
            _UpperCamelCase   :						Optional[Any]    =							1_024
            _UpperCamelCase   :						List[str]    =							24
            _UpperCamelCase   :						Optional[int]    =							16
      elif checkpoint == "medium":
            _UpperCamelCase   :						int    =							1_536
            _UpperCamelCase   :						Any    =							48
            _UpperCamelCase   :						Optional[Any]    =							24
      elif checkpoint == "large":
            _UpperCamelCase   :						int    =							2_048
            _UpperCamelCase   :						Dict    =							48
            _UpperCamelCase   :						Dict    =							32
      else:
            raise ValueError(F'''Checkpoint should be one of `[\'small\', \'medium\', \'large\']`, got {checkpoint}.'''      )
      _UpperCamelCase   :						List[Any]    =							MusicgenDecoderConfig(
          hidden_size=lowercase_		,ffn_dim=hidden_size * 4		,num_hidden_layers=lowercase_		,num_attention_heads=lowercase_		,)
      return config




@torch.no_grad()
def        lowercase__     (       lowercase_		,lowercase_=None		,lowercase_=None		,lowercase_="cpu"      )       ->		Union[str, Any]:




      """simple docstring"""

      _UpperCamelCase   :						int    =							MusicGen.get_pretrained(lowercase_		,device=lowercase_      )
      _UpperCamelCase   :						int    =							decoder_config_from_checkpoint(lowercase_      )

      _UpperCamelCase   :						int    =							fairseq_model.lm.state_dict()
      _UpperCamelCase,	_UpperCamelCase   :						Tuple    =							rename_state_dict(
          lowercase_		,hidden_size=decoder_config.hidden_size      )

      _UpperCamelCase   :						int    =							TaEncoderModel.from_pretrained("t5-base"      )
      _UpperCamelCase   :						str    =							EncodecModel.from_pretrained("facebook/encodec_32khz"      )
      _UpperCamelCase   :						Dict    =							MusicgenForCausalLM(lowercase_      ).eval()

      # load all decoder weights - expect that we'll be missing embeddings and enc-dec projection
      _UpperCamelCase,	_UpperCamelCase   :						List[str]    =							decoder.load_state_dict(lowercase_		,strict=lowercase_      )

      for key in missing_keys.copy():
            if key.startswith(("text_encoder", "audio_encoder")      ) or key in EXPECTED_MISSING_KEYS:
                  missing_keys.remove(lowercase_      )

      if len(lowercase_      ) > 0:
            raise ValueError(F'''Missing key(s) in state_dict: {missing_keys}'''      )

      if len(lowercase_      ) > 0:
            raise ValueError(F'''Unexpected key(s) in state_dict: {unexpected_keys}'''      )

      # init the composite model
      _UpperCamelCase   :						Tuple    =							MusicgenForConditionalGeneration(text_encoder=lowercase_		,audio_encoder=lowercase_		,decoder=lowercase_      )

      # load the pre-trained enc-dec projection (from the decoder state dict)
      model.enc_to_dec_proj.load_state_dict(lowercase_      )

      # check we can do a forward pass
      _UpperCamelCase   :						Any    =							torch.arange(0		,8		,dtype=torch.long      ).reshape(2		,-1      )
      _UpperCamelCase   :						List[Any]    =							input_ids.reshape(2 * 4		,-1      )

      with torch.no_grad():
            _UpperCamelCase   :						List[str]    =							model(input_ids=lowercase_		,decoder_input_ids=lowercase_      ).logits

      if logits.shape != (8, 1, 2_048):
            raise ValueError("Incorrect shape for logits"      )

      # now construct the processor
      _UpperCamelCase   :						str    =							AutoTokenizer.from_pretrained("t5-base"      )
      _UpperCamelCase   :						Union[str, Any]    =							AutoFeatureExtractor.from_pretrained("facebook/encodec_32khz"		,padding_side="left"      )

      _UpperCamelCase   :						Any    =							MusicgenProcessor(feature_extractor=lowercase_		,tokenizer=lowercase_      )

      # set the appropriate bos/pad token ids
      _UpperCamelCase   :						Optional[Any]    =							2_048
      _UpperCamelCase   :						str    =							2_048

      # set other default generation config params
      _UpperCamelCase   :						Optional[int]    =							int(30 * audio_encoder.config.frame_rate      )
      _UpperCamelCase   :						Optional[Any]    =							True
      _UpperCamelCase   :						Any    =							3.0

      if pytorch_dump_folder is not None:
            Path(lowercase_      ).mkdir(exist_ok=lowercase_      )
            logger.info(F'''Saving model {checkpoint} to {pytorch_dump_folder}'''      )
            model.save_pretrained(lowercase_      )
            processor.save_pretrained(lowercase_      )

      if repo_id:
            logger.info(F'''Pushing model {checkpoint} to {repo_id}'''      )
            model.push_to_hub(lowercase_      )
            processor.push_to_hub(lowercase_      )


if __name__ == "__main__":
  lowerCamelCase__						= argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      "--checkpoint",
      default="small",
      type=str,
      help="Checkpoint size of the MusicGen model you'd like to convert. Can be one of: `['small', 'medium', 'large']`.",
  )
  parser.add_argument(
      "--pytorch_dump_folder",
      required=True,
      default=None,
      type=str,
      help="Path to the output PyTorch model directory.",
  )
  parser.add_argument(
      "--push_to_hub", default=None, type=str, help="Where to upload the converted model on the 🤗 hub."
  )
  parser.add_argument(
      "--device", default="cpu", type=str, help="Torch device to run the conversion, either cpu or cuda."
  )

  lowerCamelCase__						= parser.parse_args()
  convert_musicgen_checkpoint(args.checkpoint, args.pytorch_dump_folder, args.push_to_hub)
