





import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(42)

snake_case__					:							Optional[int]													=     'bert-base-cased'
snake_case__					:							Union[str, Any]													=     'fp16'
snake_case__					:							Dict													=     'bf16'
snake_case__					:							Tuple													=     [FPaa, BFaa]





@require_fsdp
@require_cuda
class A_					(			_lowerCamelCase   ):




   def _lowerCAmelCase      (self	:Tuple )->					List[Any]:
         super().setUp()

         __A          =				dict(
             ACCELERATE_USE_FSDP='''true'''    ,   MASTER_ADDR='''localhost'''    ,   MASTER_PORT='''10999'''    ,   RANK='''0'''    ,   LOCAL_RANK='''0'''    ,   WORLD_SIZE='''1'''    ,   )




   def _lowerCAmelCase      (self	:int )->					List[Any]:
         from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

         for i, strategy in enumerate(_UpperCamelCase ):
               __A          =				self.dist_env.copy()
               __A          =				f"""{i + 1}"""
               __A          =				strategy
               with mockenv_context(**_UpperCamelCase ):
                     __A          =				FullyShardedDataParallelPlugin()
                     self.assertEqual(fsdp_plugin.sharding_strategy    ,   ShardingStrategy(i + 1 ) )




   def _lowerCAmelCase      (self	:Optional[Any] )->					Any:
         from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

         for i, prefetch_policy in enumerate(_UpperCamelCase ):
               __A          =				self.dist_env.copy()
               __A          =				prefetch_policy
               with mockenv_context(**_UpperCamelCase ):
                     __A          =				FullyShardedDataParallelPlugin()
                     if prefetch_policy == "NO_PREFETCH":
                           self.assertIsNone(fsdp_plugin.backward_prefetch )
                     else:
                           self.assertEqual(fsdp_plugin.backward_prefetch    ,   BackwardPrefetch(i + 1 ) )




   def _lowerCAmelCase      (self	:Tuple )->					Optional[Any]:
         from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

         for i, state_dict_type in enumerate(_UpperCamelCase ):
               __A          =				self.dist_env.copy()
               __A          =				state_dict_type
               with mockenv_context(**_UpperCamelCase ):
                     __A          =				FullyShardedDataParallelPlugin()
                     self.assertEqual(fsdp_plugin.state_dict_type    ,   StateDictType(i + 1 ) )
                     if state_dict_type == "FULL_STATE_DICT":
                           self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu )
                           self.assertTrue(fsdp_plugin.state_dict_config.ranka_only )




   def _lowerCAmelCase      (self	:List[str] )->					Optional[Any]:
         __A          =				AutoModel.from_pretrained(_UpperCamelCase )
         for policy in FSDP_AUTO_WRAP_POLICY:
               __A          =				self.dist_env.copy()
               __A          =				policy
               if policy == "TRANSFORMER_BASED_WRAP":
                     __A          =				'''BertLayer'''
               elif policy == "SIZE_BASED_WRAP":
                     __A          =				'''2000'''
               with mockenv_context(**_UpperCamelCase ):
                     __A          =				FullyShardedDataParallelPlugin()
                     fsdp_plugin.set_auto_wrap_policy(_UpperCamelCase )
                     if policy == "NO_WRAP":
                           self.assertIsNone(fsdp_plugin.auto_wrap_policy )
                     else:
                           self.assertIsNotNone(fsdp_plugin.auto_wrap_policy )

         __A          =				self.dist_env.copy()
         __A          =				'''TRANSFORMER_BASED_WRAP'''
         __A          =				'''T5Layer'''
         with mockenv_context(**_UpperCamelCase ):
               __A          =				FullyShardedDataParallelPlugin()
               with self.assertRaises(_UpperCamelCase ) as cm:
                     fsdp_plugin.set_auto_wrap_policy(_UpperCamelCase )
               self.assertTrue('''Could not find the transformer layer class to wrap in the model.''' in str(cm.exception ) )

         __A          =				self.dist_env.copy()
         __A          =				'''SIZE_BASED_WRAP'''
         __A          =				'''0'''
         with mockenv_context(**_UpperCamelCase ):
               __A          =				FullyShardedDataParallelPlugin()
               fsdp_plugin.set_auto_wrap_policy(_UpperCamelCase )
               self.assertIsNone(fsdp_plugin.auto_wrap_policy )




   def _lowerCAmelCase      (self	:str )->					Union[str, Any]:
         from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
         from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

         for mp_dtype in dtypes:
               __A          =				self.dist_env.copy()
               __A          =				mp_dtype
               with mockenv_context(**_UpperCamelCase ):
                     __A          =				Accelerator()
                     if mp_dtype == "fp16":
                           __A          =				torch.floataa
                     elif mp_dtype == "bf16":
                           __A          =				torch.bfloataa
                     __A          =				MixedPrecision(param_dtype=_UpperCamelCase    ,   reduce_dtype=_UpperCamelCase    ,   buffer_dtype=_UpperCamelCase )
                     self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy    ,   _UpperCamelCase )
                     if mp_dtype == FPaa:
                           self.assertTrue(isinstance(accelerator.scaler    ,   _UpperCamelCase ) )
                     elif mp_dtype == BFaa:
                           self.assertIsNone(accelerator.scaler )
                     AcceleratorState._reset_state(_UpperCamelCase )






   def _lowerCAmelCase      (self	:List[str] )->					Dict:
         from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

         for flag in [True, False]:
               __A          =				self.dist_env.copy()
               __A          =				str(_UpperCamelCase ).lower()
               with mockenv_context(**_UpperCamelCase ):
                     __A          =				FullyShardedDataParallelPlugin()
                     self.assertEqual(fsdp_plugin.cpu_offload    ,   CPUOffload(offload_params=_UpperCamelCase ) )





@require_fsdp
@require_multi_gpu
@slow
class A_					(			_lowerCamelCase   ):




   def _lowerCAmelCase      (self	:str )->					int:
         super().setUp()
         __A          =				0.8_2
         __A          =				[
             '''fsdp_shard_grad_op_transformer_based_wrap''',
             '''fsdp_full_shard_transformer_based_wrap''',
         ]
         __A          =				{
             '''multi_gpu_fp16''': 3200,
             '''fsdp_shard_grad_op_transformer_based_wrap_fp16''': 2000,
             '''fsdp_full_shard_transformer_based_wrap_fp16''': 1900,
             # Disabling below test as it overwhelms the RAM memory usage
             # on CI self-hosted runner leading to tests getting killed.
             # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
         }
         __A          =				160
         __A          =				160

         __A          =				inspect.getfile(accelerate.test_utils )
         __A          =				os.path.sep.join(mod_file.split(os.path.sep )[:-1] + ['''scripts''', '''external_deps'''] )




   def _lowerCAmelCase      (self	:Union[str, Any] )->					Dict:
         __A          =				os.path.join(self.test_scripts_folder    ,   '''test_performance.py''' )
         __A          =				['''accelerate''', '''launch''', '''--num_processes=2''', '''--num_machines=1''', '''--machine_rank=0''', '''--use_fsdp''']
         for config in self.performance_configs:
               __A          =				cmd.copy()
               for i, strategy in enumerate(_UpperCamelCase ):
                     if strategy.lower() in config:
                           cmd_config.append(f"""--fsdp_sharding_strategy={i+1}""" )
                           break

               if "fp32" in config:
                     cmd_config.append('''--mixed_precision=no''' )
               else:
                     cmd_config.append('''--mixed_precision=fp16''' )

               if "cpu_offload" in config:
                     cmd_config.append('''--fsdp_offload_params=True''' )

               for policy in FSDP_AUTO_WRAP_POLICY:
                     if policy.lower() in config:
                           cmd_config.append(f"""--fsdp_auto_wrap_policy={policy}""" )
                           break

               if policy == "TRANSFORMER_BASED_WRAP":
                     cmd_config.append('''--fsdp_transformer_layer_cls_to_wrap=BertLayer''' )
               elif policy == "SIZE_BASED_WRAP":
                     cmd_config.append('''--fsdp_min_num_params=2000''' )

               cmd_config.extend(
                   [
                       self.test_file_path,
                       f"""--output_dir={self.tmpdir}""",
                       f"""--performance_lower_bound={self.performance_lower_bound}""",
                   ] )
               with patch_environment(omp_num_threads=1 ):
                     execute_subprocess_async(_UpperCamelCase    ,   env=os.environ.copy() )




   def _lowerCAmelCase      (self	:str )->					List[str]:
         __A          =				os.path.join(self.test_scripts_folder    ,   '''test_checkpointing.py''' )
         __A          =				[
             '''accelerate''',
             '''launch''',
             '''--num_processes=2''',
             '''--num_machines=1''',
             '''--machine_rank=0''',
             '''--use_fsdp''',
             '''--mixed_precision=fp16''',
             '''--fsdp_transformer_layer_cls_to_wrap=BertLayer''',
         ]

         for i, strategy in enumerate(_UpperCamelCase ):
               __A          =				cmd.copy()
               cmd_config.append(f"""--fsdp_sharding_strategy={i+1}""" )
               if strategy != "FULL_SHARD":
                     continue
               __A          =				len(_UpperCamelCase )
               for state_dict_type in FSDP_STATE_DICT_TYPE:
                     __A          =				cmd_config[:state_dict_config_index]
                     cmd_config.append(f"""--fsdp_state_dict_type={state_dict_type}""" )
                     cmd_config.extend(
                         [
                             self.test_file_path,
                             f"""--output_dir={self.tmpdir}""",
                             '''--partial_train_epoch=1''',
                         ] )
                     with patch_environment(omp_num_threads=1 ):
                           execute_subprocess_async(_UpperCamelCase    ,   env=os.environ.copy() )

                     __A          =				cmd_config[:-1]
                     __A          =				os.path.join(self.tmpdir    ,   '''epoch_0''' )
                     cmd_config.extend(
                         [
                             f"""--resume_from_checkpoint={resume_from_checkpoint}""",
                         ] )
                     with patch_environment(omp_num_threads=1 ):
                           execute_subprocess_async(_UpperCamelCase    ,   env=os.environ.copy() )






   def _lowerCAmelCase      (self	:Optional[Any] )->					List[Any]:
         __A          =				os.path.join(self.test_scripts_folder    ,   '''test_peak_memory_usage.py''' )
         __A          =				[
             '''accelerate''',
             '''launch''',
             '''--num_processes=2''',
             '''--num_machines=1''',
             '''--machine_rank=0''',
         ]
         for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
               __A          =				cmd.copy()
               if "fp16" in spec:
                     cmd_config.extend(['''--mixed_precision=fp16'''] )
               else:
                     cmd_config.extend(['''--mixed_precision=no'''] )

               if "multi_gpu" in spec:
                     continue
               else:
                     cmd_config.extend(['''--use_fsdp'''] )
                     for i, strategy in enumerate(_UpperCamelCase ):
                           if strategy.lower() in spec:
                                 cmd_config.append(f"""--fsdp_sharding_strategy={i+1}""" )
                                 break

                     if "cpu_offload" in spec:
                           cmd_config.append('''--fsdp_offload_params=True''' )

                     for policy in FSDP_AUTO_WRAP_POLICY:
                           if policy.lower() in spec:
                                 cmd_config.append(f"""--fsdp_auto_wrap_policy={policy}""" )
                                 break

                     if policy == "TRANSFORMER_BASED_WRAP":
                           cmd_config.append('''--fsdp_transformer_layer_cls_to_wrap=BertLayer''' )
                     elif policy == "SIZE_BASED_WRAP":
                           cmd_config.append('''--fsdp_min_num_params=2000''' )

               cmd_config.extend(
                   [
                       self.test_file_path,
                       f"""--output_dir={self.tmpdir}""",
                       f"""--peak_memory_upper_bound={peak_mem_upper_bound}""",
                       f"""--n_train={self.n_train}""",
                       f"""--n_val={self.n_val}""",
                   ] )
               with patch_environment(omp_num_threads=1 ):
                     execute_subprocess_async(_UpperCamelCase    ,   env=os.environ.copy() )












import argparse

import fairseq
import torch

from transformers import UniSpeechSatConfig, UniSpeechSatForCTC, UniSpeechSatForPreTraining, logging


logging.set_verbosity_info()
snake_case__					:							Dict													=     logging.get_logger(__name__)

snake_case__					:							Optional[Any]													=     {
    'post_extract_proj': 'feature_projection.projection',
    'encoder.pos_conv.0': 'encoder.pos_conv_embed.conv',
    'self_attn.k_proj': 'encoder.layers.*.attention.k_proj',
    'self_attn.v_proj': 'encoder.layers.*.attention.v_proj',
    'self_attn.q_proj': 'encoder.layers.*.attention.q_proj',
    'self_attn.out_proj': 'encoder.layers.*.attention.out_proj',
    'self_attn_layer_norm': 'encoder.layers.*.layer_norm',
    'fc1': 'encoder.layers.*.feed_forward.intermediate_dense',
    'fc2': 'encoder.layers.*.feed_forward.output_dense',
    'final_layer_norm': 'encoder.layers.*.final_layer_norm',
    'encoder.layer_norm': 'encoder.layer_norm',
    'encoder.layer_norm_for_extract': 'layer_norm_for_extract',
    'w2v_model.layer_norm': 'feature_projection.layer_norm',
    'quantizer.weight_proj': 'quantizer.weight_proj',
    'quantizer.vars': 'quantizer.codevectors',
    'project_q': 'project_q',
    'final_proj': 'project_hid',
    'w2v_encoder.proj': 'lm_head',
    'label_embs_concat': 'label_embeddings_concat',
    'mask_emb': 'masked_spec_embed',
    'spk_proj': 'speaker_proj',
}
snake_case__					:							Optional[int]													=     [
    'lm_head',
    'quantizer.weight_proj',
    'quantizer.codevectors',
    'project_q',
    'project_hid',
    'label_embeddings_concat',
    'speaker_proj',
    'layer_norm_for_extract',
]



def _a				(	lowerCamelCase:   List[Any]					,				lowerCamelCase:   Any					,				lowerCamelCase:   Union[str, Any]					,				lowerCamelCase:   Any					,				lowerCamelCase:   int )	->						List[str]:







      '''simple docstring'''






      for attribute in key.split('''.''' ):
            __A          =				getattr(lowerCamelCase					,				lowerCamelCase )

      if weight_type is not None:
            __A          =				getattr(lowerCamelCase					,				lowerCamelCase ).shape
      else:
            __A          =				hf_pointer.shape

      if hf_shape != value.shape:
            raise ValueError(
                F"""Shape of hf {key + '.' + weight_type if weight_type is not None else ''} is {hf_shape}, but should be"""
                F""" {value.shape} for {full_name}""" )

      if weight_type == "weight":
            __A          =				value
      elif weight_type == "weight_g":
            __A          =				value
      elif weight_type == "weight_v":
            __A          =				value
      elif weight_type == "bias":
            __A          =				value
      else:
            __A          =				value

      logger.info(F"""{key + '.' + weight_type if weight_type is not None else ''} was initialized from {full_name}.""" )



def _a				(	lowerCamelCase:   List[str]					,				lowerCamelCase:   Optional[int] )	->						Tuple:







      '''simple docstring'''






      __A          =				[]
      __A          =				fairseq_model.state_dict()

      __A          =				hf_model.unispeech_sat.feature_extractor

      for name, value in fairseq_dict.items():
            __A          =				False
            if "conv_layers" in name:
                  load_conv_layer(
                      lowerCamelCase					,				lowerCamelCase					,				lowerCamelCase					,				lowerCamelCase					,				hf_model.config.feat_extract_norm == '''group'''					,				)
                  __A          =				True
            else:
                  for key, mapped_key in MAPPING.items():
                        __A          =				'''unispeech_sat.''' + mapped_key if mapped_key not in TOP_LEVEL_KEYS else mapped_key
                        if key in name or key.split('''w2v_model.''' )[-1] == name.split('''.''' )[0]:
                              if "layer_norm_for_extract" in name and (".".join(name.split('''.''' )[:-1] ) != key):
                                    # special case since naming is very similar
                                    continue
                              __A          =				True
                              if "*" in mapped_key:
                                    __A          =				name.split(lowerCamelCase )[0].split('''.''' )[-2]
                                    __A          =				mapped_key.replace('''*'''					,				lowerCamelCase )
                              if "weight_g" in name:
                                    __A          =				'''weight_g'''
                              elif "weight_v" in name:
                                    __A          =				'''weight_v'''
                              elif "bias" in name:
                                    __A          =				'''bias'''
                              elif "weight" in name:
                                    # TODO: don't match quantizer.weight_proj
                                    __A          =				'''weight'''
                              else:
                                    __A          =				None
                              set_recursively(lowerCamelCase					,				lowerCamelCase					,				lowerCamelCase					,				lowerCamelCase					,				lowerCamelCase )
                        continue
            if not is_used:
                  unused_weights.append(lowerCamelCase )

      logger.warning(F"""Unused weights: {unused_weights}""" )



def _a				(	lowerCamelCase:   int					,				lowerCamelCase:   Any					,				lowerCamelCase:   int					,				lowerCamelCase:   int					,				lowerCamelCase:   List[str] )	->						Union[str, Any]:







      '''simple docstring'''






      __A          =				full_name.split('''conv_layers.''' )[-1]
      __A          =				name.split('''.''' )
      __A          =				int(items[0] )
      __A          =				int(items[1] )

      if type_id == 0:
            if "bias" in name:
                  if value.shape != feature_extractor.conv_layers[layer_id].conv.bias.data.shape:
                        raise ValueError(
                            F"""{full_name} has size {value.shape}, but"""
                            F""" {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found.""" )
                  __A          =				value
                  logger.info(F"""Feat extract conv layer {layer_id} was initialized from {full_name}.""" )
            elif "weight" in name:
                  if value.shape != feature_extractor.conv_layers[layer_id].conv.weight.data.shape:
                        raise ValueError(
                            F"""{full_name} has size {value.shape}, but"""
                            F""" {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found.""" )
                  __A          =				value
                  logger.info(F"""Feat extract conv layer {layer_id} was initialized from {full_name}.""" )
      elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
            if "bias" in name:
                  if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape:
                        raise ValueError(
                            F"""{full_name} has size {value.shape}, but"""
                            F""" {feature_extractor[layer_id].layer_norm.bias.data.shape} was found.""" )
                  __A          =				value
                  logger.info(F"""Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.""" )
            elif "weight" in name:
                  if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape:
                        raise ValueError(
                            F"""{full_name} has size {value.shape}, but"""
                            F""" {feature_extractor[layer_id].layer_norm.weight.data.shape} was found.""" )
                  __A          =				value
                  logger.info(F"""Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.""" )
      else:
            unused_weights.append(lowerCamelCase )



@torch.no_grad()
def _a				(	lowerCamelCase:   Tuple					,				lowerCamelCase:   int					,				lowerCamelCase:   Optional[Any]=None					,				lowerCamelCase:   Optional[Any]=None					,				lowerCamelCase:   Optional[int]=True )	->						List[Any]:







      '''simple docstring'''






      if config_path is not None:
            __A          =				UniSpeechSatConfig.from_pretrained(lowerCamelCase )
      else:
            __A          =				UniSpeechSatConfig()

      __A          =				''''''

      if is_finetuned:
            __A          =				UniSpeechSatForCTC(lowerCamelCase )
      else:
            __A          =				UniSpeechSatForPreTraining(lowerCamelCase )

      __A   ,						__A   ,						__A          =				fairseq.checkpoint_utils.load_model_ensemble_and_task(
          [checkpoint_path]					,				arg_overrides={'''data''': '''/'''.join(dict_path.split('''/''' )[:-1] )} )
      __A          =				model[0].eval()

      recursively_load_weights(lowerCamelCase					,				lowerCamelCase )

      hf_wavavec.save_pretrained(lowerCamelCase )


if __name__ == "__main__":
       snake_case__					:							Tuple													=     argparse.ArgumentParser()
       parser.add_argument('--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model.')
       parser.add_argument('--checkpoint_path', default=None, type=str, help='Path to fairseq checkpoint')
       parser.add_argument('--dict_path', default=None, type=str, help='Path to dict of fine-tuned model')
       parser.add_argument('--config_path', default=None, type=str, help='Path to hf config.json of model to convert')
       parser.add_argument(
           '--not_finetuned', action='store_true', help='Whether the model to convert is a fine-tuned model or not'
       )
       snake_case__					:							Any													=     parser.parse_args()
       convert_unispeech_sat_checkpoint(
           args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path, args.dict_path, not args.not_finetuned
       )





