update

models/ChatTTS/config/decoder.yaml

@@ -1,3 +1,5 @@
+
+
 dim: 384
 
 decoder_config:

models/ChatTTS/config/dvae.yaml

@@ -1,3 +1,5 @@
+
+
 dim: 512
 decoder_config:
   idim: ${dim}
@@ -7,6 +9,6 @@ decoder_config:
 
 vq_config:
   dim: 1024
-  levels: [5, 5, 5, 5]
+  levels: [5,5,5,5]
   G: 2
   R: 2

models/ChatTTS/config/gpt.yaml

@@ -1,3 +1,5 @@
+
+
 num_audio_tokens: 626
 num_text_tokens: 21178
 
@@ -15,3 +17,4 @@ gpt_config:
   num_audio_tokens: 626
   num_text_tokens: null
   num_vq: 4
+

models/ChatTTS/config/path.yaml

@@ -1,3 +1,5 @@
+
+
 vocos_config_path: config/vocos.yaml
 vocos_ckpt_path: asset/Vocos.pt
 dvae_config_path: config/dvae.yaml

models/ChatTTS/config/vocos.yaml

@@ -21,4 +21,4 @@ head:
     dim: 512
     n_fft: 1024
     hop_length: 256
-    padding: center
+    padding: center

models/Denoise/audio-denoiser-512-32-v1/config.json

@@ -0,0 +1 @@
+{"scaler": {"mean": -3.027921438217163, "std": 1.9317387342453003}, "in_channels": 257, "n_fft": 512, "num_frames": 32, "exp_id": "115233"}

models/Denoise/audio-denoiser-512-32-v1/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5079784e228d2b36496f2c72f8d06015c8fb1827a81f757ec8540ca708ada7a9
+size 153639572

models/resemble-enhance/hparams.yaml

@@ -0,0 +1,38 @@
+fg_dir: !!python/object/apply:pathlib.PosixPath
+- data
+- fg
+bg_dir: !!python/object/apply:pathlib.PosixPath
+- data
+- bg
+rir_dir: !!python/object/apply:pathlib.PosixPath
+- data
+- rir
+load_fg_only: false
+wav_rate: 44100
+n_fft: 2048
+win_size: 2048
+hop_size: 420
+num_mels: 128
+stft_magnitude_min: 0.0001
+preemphasis: 0.97
+mix_alpha_range:
+- 0.2
+- 0.8
+nj: 64
+training_seconds: 3.0
+batch_size_per_gpu: 32
+min_lr: 1.0e-05
+max_lr: 0.0001
+warmup_steps: 1000
+max_steps: 1000000
+gradient_clipping: 1.0
+cfm_solver_method: midpoint
+cfm_solver_nfe: 64
+cfm_time_mapping_divisor: 4
+univnet_nc: 96
+lcfm_latent_dim: 64
+lcfm_training_mode: cfm
+lcfm_z_scale: 6
+vocoder_extra_dim: 32
+gan_training_start_step: null
+praat_augment_prob: 0.2

models/resemble-enhance/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9d035f318de3e6d919bc70cf7ad7d32b4fe92ec5cbe0b30029a27f5db07d9d6
+size 713176232

modules/Enhancer/ResembleEnhance.py

@@ -1,5 +1,6 @@
 import os
-from typing import List
+from typing import List, Literal
+from modules.devices import devices
 from modules.repos_static.resemble_enhance.enhancer.enhancer import Enhancer
 from modules.repos_static.resemble_enhance.enhancer.hparams import HParams
 from modules.repos_static.resemble_enhance.inference import inference
@@ -25,14 +26,11 @@ def load_enhancer(device: torch.device):
 
 
 class ResembleEnhance:
-    hparams: HParams
-    enhancer: Enhancer
-
     def __init__(self, device: torch.device):
         self.device = device
 
-        self.enhancer = None
-        self.hparams = None
+        self.enhancer: HParams = None
+        self.hparams: Enhancer = None
 
     def load_model(self):
         hparams = HParams.load(Path(MODELS_DIR) / "resemble-enhance")
@@ -42,9 +40,7 @@ class ResembleEnhance:
             map_location="cpu",
         )["module"]
         enhancer.load_state_dict(state_dict)
-        enhancer.eval()
-        enhancer.to(self.device)
-        enhancer.denoiser.to(self.device)
+        enhancer.to(self.device).eval()
 
         self.hparams = hparams
         self.enhancer = enhancer
@@ -63,7 +59,7 @@ class ResembleEnhance:
         sr,
         device,
         nfe=32,
-        solver="midpoint",
+        solver: Literal["midpoint", "rk4", "euler"] = "midpoint",
         lambd=0.5,
         tau=0.5,
     ) -> tuple[torch.Tensor, int]:
@@ -83,34 +79,51 @@ class ResembleEnhance:
 
 if __name__ == "__main__":
     import torchaudio
-    from modules.models import load_chat_tts
-
-    load_chat_tts()
+    import gradio as gr
 
     device = torch.device("cuda")
-    ench = ResembleEnhance(device)
-    ench.load_model()
 
-    wav, sr = torchaudio.load("test.wav")
+    # def enhance(file):
+    #     print(file)
+    #     ench = load_enhancer(device)
+    #     dwav, sr = torchaudio.load(file)
+    #     dwav = dwav.mean(dim=0).to(device)
+    #     enhanced, e_sr = ench.enhance(dwav, sr)
+    #     return e_sr, enhanced.cpu().numpy()
+
+    # # 随便一个示例
+    # gr.Interface(
+    #     fn=enhance, inputs=[gr.Audio(type="filepath")], outputs=[gr.Audio()]
+    # ).launch()
+
+    # load_chat_tts()
+
+    # ench = load_enhancer(device)
+
+    # devices.torch_gc()
+
+    # wav, sr = torchaudio.load("test.wav")
 
-    print(wav.shape, type(wav), sr, type(sr))
-    exit()
+    # print(wav.shape, type(wav), sr, type(sr))
+    # # exit()
 
-    wav = wav.squeeze(0).cuda()
+    # wav = wav.squeeze(0).cuda()
 
-    print(wav.device)
+    # print(wav.device)
 
-    denoised, d_sr = ench.denoise(wav.cpu(), sr, device)
-    denoised = denoised.unsqueeze(0)
-    print(denoised.shape)
-    torchaudio.save("denoised.wav", denoised, d_sr)
+    # denoised, d_sr = ench.denoise(wav, sr)
+    # denoised = denoised.unsqueeze(0)
+    # print(denoised.shape)
+    # torchaudio.save("denoised.wav", denoised.cpu(), d_sr)
 
-    for solver in ("midpoint", "rk4", "euler"):
-        for lambd in (0.1, 0.5, 0.9):
-            for tau in (0.1, 0.5, 0.9):
-                enhanced, e_sr = ench.enhance(
-                    wav.cpu(), sr, device, solver=solver, lambd=lambd, tau=tau, nfe=128
-                )
-                enhanced = enhanced.unsqueeze(0)
-                print(enhanced.shape)
-                torchaudio.save(f"enhanced_{solver}_{lambd}_{tau}.wav", enhanced, e_sr)
+    # for solver in ("midpoint", "rk4", "euler"):
+    #     for lambd in (0.1, 0.5, 0.9):
+    #         for tau in (0.1, 0.5, 0.9):
+    #             enhanced, e_sr = ench.enhance(
+    #                 wav, sr, solver=solver, lambd=lambd, tau=tau, nfe=128
+    #             )
+    #             enhanced = enhanced.unsqueeze(0)
+    #             print(enhanced.shape)
+    #             torchaudio.save(
+    #                 f"enhanced_{solver}_{lambd}_{tau}.wav", enhanced.cpu(), e_sr
+    #             )

modules/generate_audio.py

@@ -72,7 +72,7 @@ def generate_audio_batch(
     }
 
     if isinstance(spk, int):
-        with SeedContext(spk):
+        with SeedContext(spk, True):
             params_infer_code["spk_emb"] = chat_tts.sample_random_speaker()
         logger.info(("spk", spk))
     elif isinstance(spk, Speaker):
@@ -94,7 +94,7 @@ def generate_audio_batch(
         }
     )
 
-    with SeedContext(infer_seed):
+    with SeedContext(infer_seed, True):
         wavs = chat_tts.generate_audio(
             texts, params_infer_code, use_decoder=use_decoder
         )

modules/repos_static/resemble_enhance/enhancer/enhancer.py

@@ -73,8 +73,8 @@ class Enhancer(nn.Module):
             )
             self._load_pretrained(pretrained_path)
 
-        logger.info(f"{self.__class__.__name__} summary")
-        logger.info(f"{self.summarize()}")
+        # logger.info(f"{self.__class__.__name__} summary")
+        # logger.info(f"{self.summarize()}")
 
     def _load_pretrained(self, path):
         # Clone is necessary as otherwise it holds a reference to the original model

modules/speaker.py

@@ -11,7 +11,7 @@ import uuid
 
 def create_speaker_from_seed(seed):
     chat_tts = models.load_chat_tts()
-    with SeedContext(seed):
+    with SeedContext(seed, True):
         emb = chat_tts.sample_random_speaker()
     return emb
 

modules/utils/SeedContext.py

@@ -7,15 +7,17 @@ import logging
 logger = logging.getLogger(__name__)
 
 
-def deterministic(seed=0):
+def deterministic(seed=0, cudnn_deterministic=False):
     random.seed(seed)
     np.random.seed(seed)
     torch_rn = rng.convert_np_to_torch(seed)
     torch.manual_seed(torch_rn)
     if torch.cuda.is_available():
         torch.cuda.manual_seed_all(torch_rn)
-    torch.backends.cudnn.deterministic = True
-    torch.backends.cudnn.benchmark = False
+
+        if cudnn_deterministic:
+            torch.backends.cudnn.deterministic = True
+            torch.backends.cudnn.benchmark = False
 
 
 def is_numeric(obj):
@@ -36,7 +38,7 @@ def is_numeric(obj):
 
 
 class SeedContext:
-    def __init__(self, seed):
+    def __init__(self, seed, cudnn_deterministic=False):
         assert is_numeric(seed), "Seed must be an number."
 
         try:
@@ -45,6 +47,7 @@ class SeedContext:
             raise ValueError(f"Seed must be an integer, but: {type(seed)}")
 
         self.seed = seed
+        self.cudnn_deterministic = cudnn_deterministic
         self.state = None
 
         if isinstance(seed, str) and seed.isdigit():
@@ -57,10 +60,16 @@ class SeedContext:
             self.seed = random.randint(0, 2**32 - 1)
 
     def __enter__(self):
-        self.state = (torch.get_rng_state(), random.getstate(), np.random.get_state())
+        self.state = (
+            torch.get_rng_state(),
+            random.getstate(),
+            np.random.get_state(),
+            torch.backends.cudnn.deterministic,
+            torch.backends.cudnn.benchmark,
+        )
 
         try:
-            deterministic(self.seed)
+            deterministic(self.seed, cudnn_deterministic=self.cudnn_deterministic)
         except Exception as e:
             # raise ValueError(
             #     f"Seed must be an integer, but: <{type(self.seed)}> {self.seed}"
@@ -73,6 +82,8 @@ class SeedContext:
         torch.set_rng_state(self.state[0])
         random.setstate(self.state[1])
         np.random.set_state(self.state[2])
+        torch.backends.cudnn.deterministic = self.state[3]
+        torch.backends.cudnn.benchmark = self.state[4]
 
 
 if __name__ == "__main__":

modules/webui/app.py

@@ -16,11 +16,6 @@ from modules.webui.readme_tab import create_readme_tab
 
 logger = logging.getLogger(__name__)
 
-logging.basicConfig(
-    level=os.getenv("LOG_LEVEL", "INFO"),
-    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-)
-
 
 def webui_init():
     # fix: If the system proxy is enabled in the Windows system, you need to skip these

modules/webui/speaker/speaker_creator.py

@@ -61,7 +61,7 @@ def create_spk_from_seed(
     desc: str,
 ):
     chat_tts = load_chat_tts()
-    with SeedContext(seed):
+    with SeedContext(seed, True):
         emb = chat_tts.sample_random_speaker()
     spk = Speaker(seed=-2, name=name, gender=gender, describe=desc)
     spk.emb = emb
@@ -118,7 +118,7 @@ def speaker_creator_ui():
                     with gr.Row():
                         current_seed = gr.Label(label="Current Seed", value=-1)
                         with gr.Column(scale=4):
-                            output_audio = gr.Audio(label="Output Audio")
+                            output_audio = gr.Audio(label="Output Audio", format="mp3")
 
         test_voice_btn.click(
             fn=test_spk_voice,

modules/webui/speaker/speaker_merger.py

@@ -204,7 +204,9 @@ def create_speaker_merger():
                                     value="说话人合并测试 123456789 [uv_break] ok, test done [lbreak]",
                                 )
 
-                                output_audio = gr.Audio(label="Output Audio")
+                                output_audio = gr.Audio(
+                                    label="Output Audio", format="mp3"
+                                )
 
         with gr.Column(scale=1):
             with gr.Group():

modules/webui/ssml_tab.py

@@ -44,7 +44,7 @@ def create_ssml_interface():
                     inputs=[ssml_input],
                 )
 
-    ssml_output = gr.Audio(label="Generated Audio")
+    ssml_output = gr.Audio(label="Generated Audio", format="mp3")
 
     ssml_button.click(
         synthesize_ssml,

modules/webui/tts_tab.py

@@ -204,7 +204,7 @@ def create_tts_interface():
 
             with gr.Group():
                 gr.Markdown("🎨Output")
-                tts_output = gr.Audio(label="Generated Audio")
+                tts_output = gr.Audio(label="Generated Audio", format="mp3")
         with gr.Column(scale=1):
             with gr.Group():
                 gr.Markdown("🎶Refiner")
@@ -220,10 +220,9 @@ def create_tts_interface():
                     value=False, label="Disable Normalize"
                 )
 
-                # FIXME: 不知道为啥，就是非常慢，单独调脚本是很快的
-                with gr.Group(visible=webui_config.experimental):
+                with gr.Group():
                     gr.Markdown("💪🏼Enhance")
-                    enable_enhance = gr.Checkbox(value=False, label="Enable Enhance")
+                    enable_enhance = gr.Checkbox(value=True, label="Enable Enhance")
                     enable_de_noise = gr.Checkbox(value=False, label="Enable De-noise")
                 tts_button = gr.Button(
                     "🔊Generate Audio",

modules/webui/webui_utils.py

@@ -1,3 +1,4 @@
+import io
 from typing import Union
 import numpy as np
 
@@ -23,6 +24,9 @@ from modules import refiner
 from modules.utils import audio
 from modules.SentenceSplitter import SentenceSplitter
 
+from pydub import AudioSegment
+import torch.profiler
+
 
 def get_speakers():
     return speaker_mgr.list_speakers()
@@ -67,22 +71,23 @@ def segments_length_limit(
 @torch.inference_mode()
 @spaces.GPU
 def apply_audio_enhance(audio_data, sr, enable_denoise, enable_enhance):
-    audio_data = torch.from_numpy(audio_data).float().squeeze().cpu()
-    if enable_denoise or enable_enhance:
-        enhancer = load_enhancer(devices.device)
-        if enable_denoise:
-            audio_data, sr = enhancer.denoise(audio_data, sr, devices.device)
-        if enable_enhance:
-            audio_data, sr = enhancer.enhance(
-                audio_data,
-                sr,
-                devices.device,
-                tau=0.9,
-                nfe=64,
-                solver="euler",
-                lambd=0.5,
-            )
-    audio_data = audio_data.cpu().numpy()
+    if not enable_denoise and not enable_enhance:
+        return audio_data, sr
+
+    device = devices.device
+    # NOTE: 这里很奇怪按道理得放到 device 上，但是 enhancer 做 chunk 的时候会报错...所以得 cpu()
+    tensor = torch.from_numpy(audio_data).float().squeeze().cpu()
+    enhancer = load_enhancer(device)
+
+    if enable_enhance:
+        lambd = 0.9 if enable_denoise else 0.1
+        tensor, sr = enhancer.enhance(
+            tensor, sr, tau=0.5, nfe=64, solver="rk4", lambd=lambd, device=device
+        )
+    elif enable_denoise:
+        tensor, sr = enhancer.denoise(tensor, sr)
+
+    audio_data = tensor.cpu().numpy()
     return audio_data, int(sr)
 
 
@@ -111,10 +116,12 @@ def synthesize_ssml(ssml: str, batch_size=4):
     audio_segments = synthesize.synthesize_segments(segments)
     combined_audio = combine_audio_segments(audio_segments)
 
-    return audio.pydub_to_np(combined_audio)
+    sr, audio_data = audio.pydub_to_np(combined_audio)
 
+    return sr, audio_data
 
-@torch.inference_mode()
+
+# @torch.inference_mode()
 @spaces.GPU
 def tts_generate(
     text,
@@ -186,7 +193,6 @@ def tts_generate(
     audio_data, sample_rate = apply_audio_enhance(
         audio_data, sample_rate, enable_denoise, enable_enhance
     )
-
     audio_data = audio.audio_to_int16(audio_data)
     return sample_rate, audio_data
 

webui.py

@@ -1,4 +1,11 @@
 import os
+import logging
+
+# logging.basicConfig(
+#     level=os.getenv("LOG_LEVEL", "INFO"),
+#     format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+# )
+
 from modules.devices import devices
 from modules.utils import env
 from modules.webui import webui_config