| { | |
| "reference_unet": { | |
| "_class_name": "UNet2DConditionModel", | |
| "act_fn": "silu", | |
| "attention_head_dim": 8, | |
| "block_out_channels": [ | |
| 320, | |
| 640, | |
| 1280, | |
| 1280 | |
| ], | |
| "center_input_sample": false, | |
| "cross_attention_dim": 768, | |
| "down_block_types": [ | |
| "CrossAttnDownBlock2D", | |
| "CrossAttnDownBlock2D", | |
| "CrossAttnDownBlock2D", | |
| "DownBlock2D" | |
| ], | |
| "downsample_padding": 1, | |
| "flip_sin_to_cos": true, | |
| "freq_shift": 0, | |
| "in_channels": 4, | |
| "layers_per_block": 2, | |
| "mid_block_scale_factor": 1, | |
| "norm_eps": 1e-05, | |
| "norm_num_groups": 32, | |
| "out_channels": 4, | |
| "sample_size": 64, | |
| "up_block_types": [ | |
| "UpBlock2D", | |
| "CrossAttnUpBlock2D", | |
| "CrossAttnUpBlock2D", | |
| "CrossAttnUpBlock2D" | |
| ] | |
| }, | |
| "denoising_unet": { | |
| "_class_name": "UNet2DConditionModel", | |
| "act_fn": "silu", | |
| "attention_head_dim": 8, | |
| "block_out_channels": [ | |
| 320, | |
| 640, | |
| 1280, | |
| 1280 | |
| ], | |
| "center_input_sample": false, | |
| "cross_attention_dim": 768, | |
| "down_block_types": [ | |
| "CrossAttnDownBlock3D", | |
| "CrossAttnDownBlock3D", | |
| "CrossAttnDownBlock3D", | |
| "DownBlock3D" | |
| ], | |
| "downsample_padding": 1, | |
| "flip_sin_to_cos": true, | |
| "freq_shift": 0, | |
| "in_channels": 4, | |
| "layers_per_block": 2, | |
| "mid_block_scale_factor": 1, | |
| "norm_eps": 1e-05, | |
| "norm_num_groups": 32, | |
| "out_channels": 4, | |
| "sample_size": 64, | |
| "up_block_types": [ | |
| "UpBlock3D", | |
| "CrossAttnUpBlock3D", | |
| "CrossAttnUpBlock3D", | |
| "CrossAttnUpBlock3D" | |
| ], | |
| "mid_block_type": "UNetMidBlock3DCrossAttn", | |
| "use_inflated_groupnorm": true, | |
| "unet_use_cross_frame_attention": false, | |
| "unet_use_temporal_attention": false, | |
| "use_motion_module": true, | |
| "motion_module_resolutions": [ | |
| 1, | |
| 2, | |
| 4, | |
| 8 | |
| ], | |
| "motion_module_mid_block": true, | |
| "motion_module_decoder_only": false, | |
| "motion_module_type": "Vanilla", | |
| "motion_module_kwargs": { | |
| "num_attention_heads": 8, | |
| "num_transformer_block": 1, | |
| "attention_block_types": [ | |
| "Temporal_Self", | |
| "Temporal_Self" | |
| ], | |
| "temporal_position_encoding": true, | |
| "temporal_position_encoding_max_len": 32, | |
| "temporal_attention_dim_div": 1 | |
| } | |
| }, | |
| "vae": { | |
| "_class_name": "AutoencoderKL", | |
| "act_fn": "silu", | |
| "block_out_channels": [ | |
| 128, | |
| 256, | |
| 512, | |
| 512 | |
| ], | |
| "down_block_types": [ | |
| "DownEncoderBlock2D", | |
| "DownEncoderBlock2D", | |
| "DownEncoderBlock2D", | |
| "DownEncoderBlock2D" | |
| ], | |
| "in_channels": 3, | |
| "latent_channels": 4, | |
| "layers_per_block": 2, | |
| "norm_num_groups": 32, | |
| "out_channels": 3, | |
| "sample_size": 256, | |
| "up_block_types": [ | |
| "UpDecoderBlock2D", | |
| "UpDecoderBlock2D", | |
| "UpDecoderBlock2D", | |
| "UpDecoderBlock2D" | |
| ] | |
| }, | |
| "image_encoder": { | |
| "architectures": [ | |
| "CLIPVisionModelWithProjection" | |
| ], | |
| "attention_dropout": 0, | |
| "dropout": 0, | |
| "hidden_act": "quick_gelu", | |
| "hidden_size": 1024, | |
| "image_size": 224, | |
| "initializer_factor": 1, | |
| "initializer_range": 0.02, | |
| "intermediate_size": 4096, | |
| "layer_norm_eps": 0.00001, | |
| "model_type": "clip_vision_model", | |
| "num_attention_heads": 16, | |
| "num_channels": 3, | |
| "num_hidden_layers": 24, | |
| "patch_size": 14, | |
| "projection_dim": 768, | |
| "torch_dtype": "float32" | |
| }, | |
| "pose_guider": { | |
| "noise_latent_channels": 320, | |
| "use_ca": true | |
| }, | |
| "audio_mesher": { | |
| "_class_name": "Audio2Mesh", | |
| "attention_dropout": 0.1, | |
| "bos_token_id": 1, | |
| "codevector_dim": 256, | |
| "contrastive_logits_temperature": 0.1, | |
| "conv_bias": false, | |
| "conv_dim": [512,512,512,512,512,512,512], | |
| "conv_kernel": [10,3,3,3,3,2,2], | |
| "conv_stride": [5,2,2,2,2,2,2], | |
| "ctc_loss_reduction": "sum", | |
| "ctc_zero_infinity": false, | |
| "diversity_loss_weight": 0.1, | |
| "do_stable_layer_norm": false, | |
| "eos_token_id": 2, | |
| "feat_extract_activation": "gelu", | |
| "feat_extract_dropout": 0.0, | |
| "feat_extract_norm": "group", | |
| "feat_proj_dropout": 0.1, | |
| "feat_quantizer_dropout": 0.0, | |
| "final_dropout": 0.1, | |
| "gradient_checkpointing": false, | |
| "hidden_act": "gelu", | |
| "hidden_dropout": 0.1, | |
| "hidden_dropout_prob": 0.1, | |
| "hidden_size": 768, | |
| "initializer_range": 0.02, | |
| "intermediate_size": 3072, | |
| "layer_norm_eps": 1e-05, | |
| "layerdrop": 0.1, | |
| "mask_feature_length": 10, | |
| "mask_feature_prob": 0.0, | |
| "mask_time_length": 10, | |
| "mask_time_prob": 0.05, | |
| "model_type": "wav2vec2", | |
| "num_attention_heads": 12, | |
| "num_codevector_groups": 2, | |
| "num_codevectors_per_group": 320, | |
| "num_conv_pos_embedding_groups": 16, | |
| "num_conv_pos_embeddings": 128, | |
| "num_feat_extract_layers": 7, | |
| "num_hidden_layers": 12, | |
| "num_negatives": 100, | |
| "pad_token_id": 0, | |
| "proj_codevector_dim": 256, | |
| "vocab_size": 32, | |
| "feature_size": 1, | |
| "sampling_rate": 16000, | |
| "padding_value": 0.0, | |
| "padding_side": "right", | |
| "do_normalize": true, | |
| "return_attention_mask": false, | |
| "out_dim": 1404, | |
| "latent_dim": 512, | |
| "use_final_features": true, | |
| "zero_init": false | |
| }, | |
| "scheduler": { | |
| "num_train_timesteps": 1000, | |
| "beta_start": 0.00085, | |
| "beta_end": 0.012, | |
| "beta_schedule": "linear", | |
| "steps_offset": 1, | |
| "clip_sample": false, | |
| "rescale_betas_zero_snr": true, | |
| "timestep_spacing": "trailing", | |
| "prediction_type": "v_prediction" | |
| } | |
| } | |