Upload 70 files

.gitattributes

@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+denoiser/g_best filter=lfs diff=lfs merge=lfs -text
+ttv_v1/monotonic_align/build/temp.linux-x86_64-3.7/core.o filter=lfs diff=lfs merge=lfs -text
+ttv_v1/monotonic_align/build/temp.linux-x86_64-3.8/core.o filter=lfs diff=lfs merge=lfs -text
+ttv_v1/monotonic_align/build/temp.linux-x86_64-3.9/core.o filter=lfs diff=lfs merge=lfs -text
+ttv_v1/monotonic_align/monotonic_align/core.cpython-38-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+ttv_v1/monotonic_align/monotonic_align/core.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Sang-Hoon Lee
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Mels_preprocess.py

@@ -0,0 +1,42 @@
+import numpy as np
+
+np.random.seed(1234)
+
+import torch
+from torchaudio.transforms import MelSpectrogram, Spectrogram, MelScale
+
+class MelSpectrogramFixed(torch.nn.Module):
+    """In order to remove padding of torchaudio package + add log scale."""
+
+    def __init__(self, **kwargs):
+        super(MelSpectrogramFixed, self).__init__()
+        self.torchaudio_backend = MelSpectrogram(**kwargs)
+
+    def forward(self, x):
+        outputs = torch.log(self.torchaudio_backend(x) + 0.001)
+
+        return outputs[..., :-1]
+
+class SpectrogramFixed(torch.nn.Module):
+    """In order to remove padding of torchaudio package + add log10 scale."""
+
+    def __init__(self, **kwargs):
+        super(SpectrogramFixed, self).__init__()
+        self.torchaudio_backend = Spectrogram(**kwargs)
+
+    def forward(self, x):
+        outputs = self.torchaudio_backend(x)
+
+        return outputs[..., :-1]
+
+class MelfilterFixed(torch.nn.Module):
+    """In order to remove padding of torchaudio package + add log10 scale."""
+
+    def __init__(self, **kwargs):
+        super(MelfilterFixed, self).__init__()
+        self.torchaudio_backend = MelScale(**kwargs)
+
+    def forward(self, x):
+        outputs = torch.log(self.torchaudio_backend(x) + 0.001)
+
+        return outputs

README.md

@@ -1,13 +1,269 @@
----
-title: HierSpeech TTS
-emoji: 📊
-colorFrom: indigo
-colorTo: gray
-sdk: gradio
-sdk_version: 4.4.0
-app_file: app.py
-pinned: false
-license: cc-by-nc-4.0
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# HierSpeech++: Bridging the Gap between Semantic and Acoustic Representation by Hierarchical Variational Inference for Zero-shot Speech Synthesis
+The official implementation of HierSpeech2 | [Paper]() | [Demo page](https://sh-lee-prml.github.io/HierSpeechpp-demo/) | [Checkpoint](https://drive.google.com/drive/folders/1-L_90BlCkbPyKWWHTUjt5Fsu3kz0du0w?usp=sharing) | 
+
+**Sang-Hoon Lee, Ha-Yeong Choi, Seung-Bin Kim, Seong-Whan Lee**
+
+ Department of Artificial Intelligence, Korea University, Seoul, Korea  
+
+## Abstract
+![image](https://github.com/sh-lee-prml/HierSpeechpp/assets/56749640/732bc183-bf11-4f32-84a9-e9eab8190e1a)
+<details> 
+<summary> [Abs.] Sorry for too long abstract😅 </summary>
+
+
+Recently, large-scale language models (LLM)-based speech synthesis has shown a significant performance in zero-shot speech synthesis. However, they require a large-scale data and even suffer from the same limitation of previous autoregressive speech models such as slow inference speed and lack of robustness. Following the previous powerful end-to-end text-to-speech framework of VITS (but now that's what we call classical), this paper proposes HierSpeech++, a fast and strong zero-shot speech synthesizer for text-to-speech (TTS) and voice conversion (VC). In the previous our works (HierSpeech and HierVST), we verified that hierarchical speech synthesis frameworks could significantly improve the robustness and expressiveness of the synthetic speech by adopting hierarchical variational autoencoder and leveraging self-supervised speech represenation as an additional linguistic information to bridge an information gap between text and speech. In this work, we once again significantly improve the naturalness and speaker similarity of the synthetic speech even in the zero-shot speech synthesis scenarios. We first introduce multi-audio acoustic encoder for the enhanced acoustics posterior, and adopt a hierarchical adaptive waveform generator with conditional/unconditional generation. Second, we additionally utilize a F0 information and introduce source-filter theory-based multi-path semantic encoder for speaker-agnostic and speaker-related semantic representation. We also leverage hierarchical variational autoencoder to connect multiple representations, and present a BiT-Flow which is a bidirectional normalizing flow Transformer networks with AdaLN-Zero for better speaker adaptation and train-inference mismatch reduction. Without any text transcripts, we only utilize the speech dataset to train the speech synthesizer for data flexibility. For text-to-speech, we introduce text-to-vec (TTV) frameworks to generate a self-supervised speech representation and F0 representation from text representation and prosody prompt. Then, the speech synthesizer of HierSpeech++ generates the speech from generated vector, F0, and voice prompt. In addition, we propose the high-efficient speech super-resolution framework which can upsample the waveform audio from 16 kHz to 48 kHz, and this facilitate training the speech synthesizer in that we can use easily available low-resolution (16 kHz) speech data for scaling-up. The experimental results demonstrated that hierarchical variational autoencoder could be a strong zero-shot speech synthesizer by beating LLM-based models and diffusion-based models for TTS and VC tasks. Furthermore, we also verify the data efficiency in that our model trained with a small dataset still shows a better performance in both naturalness and similarity than other models trained with large-scale dataset. Moreover, we achieve the first human-level quality in zero-shot speech synthesis.
+</details>
+
+This repository contains:
+
+- 🪐 A PyTorch implementation of HierSpeech++ (TTV, Hierarchical Speech Synthesizer, SpeechSR)
+- ⚡️ Pre-trained HierSpeech++ models trained on LibriTTS (Train-460, Train-960, and more dataset)
+
+<!--
+- 💥 A Colab notebook for running pre-trained HierSpeech++ models (Soon..)
+🛸 A HierSpeech++ training script (Will be released soon)
+-->
+## Previous Our Works
+- [1] HierSpeech: Bridging the Gap between Text and Speech by Hierarchical Variational Inference using Self-supervised Representations for Speech Synthesis
+- [2] HierVST: Hierarchical Adaptive Zero-shot Voice Style Transfer
+
+This paper is an extenstion version of above papers.
+
+## Todo
+### Hierarchical Speech Synthesizer
+- [x] HierSpeechpp-Backbone
+<!--
+- [ ] HierSpeech-Lite (Fast and Efficient Zero-shot Speech Synthesizer)
+- [ ] HierSinger (Zero-shot Singing Voice Synthesizer)
+- [ ] HierSpeech2-24k-Large-Full (For High-resolutional and High-quality Speech Synthesizer)
+- [ ] HierSpeech2-48k-Large-Full (For Industrial-level High-resolution and High-quality Speech Synthesizer)
+-->
+### Text-to-Vec (TTV)
+- [x] TTV-v1 (LibriTTS-train-960)
+- [ ] TTV-v2 (We are currently training a multi-lingual TTV model)  
+<!--
+- [ ] Hierarchical Text-to-Vec (For Much More Expressive Text-to-Speech)
+-->
+### Speech Super-resolution (16k --> 24k or 48k) 
+- [x] SpeechSR-24k
+- [x] SpeechSR-48k
+### Training code (Will be released after paper acceptance)
+- [ ] TTV
+- [ ] Hierarchical Speech Synthesizer
+- [ ] SpeechSR
+## Getting Started
+
+### Pre-requisites
+0. Pytorch >=1.13 and torchaudio >= 0.13
+1. Install requirements
+```
+pip install -r requirements.txt
+```
+2. Install Phonemizer
+```
+pip install phonemizer
+sudo apt-get install espeak-ng
+```
+   
+## Checkpoint [[Download]](https://drive.google.com/drive/folders/1-L_90BlCkbPyKWWHTUjt5Fsu3kz0du0w?usp=sharing)
+### Hierarchical Speech Synthesizer
+| Model |Sampling Rate|Params|Dataset|Hour|Speaker|Checkpoint|
+|------|:---:|:---:|:---:|:---:|:---:|:---:|
+| HierSpeech2|16 kHz|97M| LibriTTS (train-460) |245|1,151|[[Download]](https://drive.google.com/drive/folders/14FTu0ZWux0zAD7ev4O1l6lKslQcdmebL?usp=sharing)|
+| HierSpeech2|16 kHz|97M| LibriTTS (train-960) |555|2,311|[[Download]](https://drive.google.com/drive/folders/1sFQP-8iS8z9ofCkE7szXNM_JEy4nKg41?usp=drive_link)|
+| HierSpeech2|16 kHz|97M| LibriTTS (train-960), Libri-light (Small, Medium), Expresso, MMS(Kor), NIKL(Kor)|2,796| 7,299 |[[Download]](https://drive.google.com/drive/folders/14jaDUBgrjVA7bCODJqAEirDwRlvJe272?usp=drive_link)|
+
+<!--
+| HierSpeech2-Lite|16 kHz|-| LibriTTS (train-960))  |-|
+| HierSpeech2-Lite|16 kHz|-| LibriTTS (train-960) NIKL, AudioBook-Korean)  |-|
+| HierSpeech2-Large-CL|16 kHz|200M| LibriTTS (train-960), Libri-Light, NIKL, AudioBook-Korean, Japanese, Chinese, CSS, MLS)  |-|
+-->
+
+### TTV
+| Model |Language|Params|Dataset|Hour|Speaker|Checkpoint|
+|------|:---:|:---:|:---:|:---:|:---:|:---:|
+| TTV |Eng|107M| LibriTTS (train-960) |555|2,311|[[Download]](https://drive.google.com/drive/folders/1QiFFdPhqhiLFo8VXc0x7cFHKXArx7Xza?usp=drive_link)|
+
+
+<!--
+| TTV |Kor|100M| NIKL |114|118|-|
+| TTV |Eng|50M| LibriTTS (train-960) |555|2,311|-|
+| TTV-Large |Eng|100M| LibriTTS (train-960) |555|2,311|-|
+| TTV-Lite |Eng|10M| LibriTTS (train-960) |555|2,311|-|
+| TTV |Kor|50M| NIKL |114|118|-|
+-->
+### SpeechSR
+| Model |Sampling Rate|Params|Dataset |Checkpoint|
+|------|:---:|:---:|:---:|:---:|
+| SpeechSR-24k |16kHz --> 24 kHz|0.13M| LibriTTS (train-960), MMS (Kor) |[speechsr24k](https://github.com/sh-lee-prml/HierSpeechpp/blob/main/speechsr24k/G_340000.pth)|
+| SpeechSR-48k |16kHz --> 48 kHz|0.13M| MMS (Kor), Expresso (Eng), VCTK (Eng)|[speechsr48k](https://github.com/sh-lee-prml/HierSpeechpp/blob/main/speechsr48k/G_100000.pth)|
+
+## Text-to-Speech
+```
+sh inference.sh
+
+# --ckpt "logs/hierspeechpp_libritts460/hierspeechpp_lt460_ckpt.pth" \ LibriTTS-460
+# --ckpt "logs/hierspeechpp_libritts960/hierspeechpp_lt960_ckpt.pth" \ LibriTTS-960
+# --ckpt "logs/hierspeechpp_eng_kor/hierspeechpp_v1_ckpt.pth" \ Large_v1 epoch 60 (paper version)
+# --ckpt "logs/hierspeechpp_eng_kor/hierspeechpp_v2_ckpt.pth" \ Large_v2 epoch 110 (08. Nov. 2023)
+
+CUDA_VISIBLE_DEVICES=0 python3 inference.py \
+                --ckpt "logs/hierspeechpp_eng_kor/hierspeechpp_v2_ckpt.pth" \
+                --ckpt_text2w2v "logs/ttv_libritts_v1/ttv_lt960_ckpt.pth" \
+                --output_dir "tts_results_eng_kor_v2" \
+                --noise_scale_vc "0.333" \
+                --noise_scale_ttv "0.333" \
+                --denoise_ratio "0"
+
+```
+- For better robustness, we recommend a noise_scale of 0.333
+- For better expressiveness, we recommend a noise_scale of 0.667
+- Find your best parameters for your style prompt 😵
+### Noise Control 
+```
+# without denoiser
+--denoise_ratio "0"
+# with denoiser
+--denoise_ratio "1"
+# Mixup (Recommended 0.6~0.8)
+--denoise_rate "0.8" 
+```
+## Voice Conversion
+- This method only utilize a hierarchical speech synthesizer for voice conversion. 
+```
+sh inference_vc.sh
+
+# --ckpt "logs/hierspeechpp_libritts460/hierspeechpp_lt460_ckpt.pth" \ LibriTTS-460
+# --ckpt "logs/hierspeechpp_libritts960/hierspeechpp_lt960_ckpt.pth" \ LibriTTS-960
+# --ckpt "logs/hierspeechpp_eng_kor/hierspeechpp_v1_ckpt.pth" \ Large_v1 epoch 60 (paper version)
+# --ckpt "logs/hierspeechpp_eng_kor/hierspeechpp_v2_ckpt.pth" \ Large_v2 epoch 110 (08. Nov. 2023)
+
+CUDA_VISIBLE_DEVICES=0 python3 inference_vc.py \
+                --ckpt "logs/hierspeechpp_eng_kor/hierspeechpp_v2_ckpt.pth" \
+                --output_dir "vc_results_eng_kor_v2" \
+                --noise_scale_vc "0.333" \
+                --noise_scale_ttv "0.333" \
+                --denoise_ratio "0"
+```
+- For better robustness, we recommend a noise_scale of 0.333
+- For better expressiveness, we recommend a noise_scale of 0.667
+- Find your best parameters for your style prompt 😵
+- Voice Conversion is vulnerable to noisy target prompt so we recommend to utilize a denoiser with noisy prompt
+- For noisy source speech, a wrong F0 may be extracted by YAPPT resulting in a quality degradation.
+
+
+## Speech Super-resolution
+- SpeechSR-24k and SpeechSR-48 are provided in TTS pipeline. If you want to use SpeechSR only, please refer inference_speechsr.py.
+- If you change the output resolution, add this
+```
+--output_sr "48000" # Default
+--output_sr "24000" # 
+--output_sr "16000" # without super-resolution.
+```
+## Speech Denoising for Noise-free Speech Synthesis (Only used in Speaker Encoder during Inference)
+- For denoised style prompt, we utilize a denoiser [(MP-SENet)](https://github.com/yxlu-0102/MP-SENet).
+- When using a long reference audio, there is an out-of-memory issue with this model so we have a plan to learn a memory efficient speech denoiser in the future.
+- If you have a problem, we recommend to use a clean reference audio or denoised audio before TTS pipeline or denoise the audio with cpu (but this will be slow😥). 
+
+## TTV-v2
+- TTV-v1 is a simple model which is very slightly modified from VITS. Although this simple TTV could synthesize a speech with high-quality and high speaker similarity, we thought that there is room for improvement in terms of expressiveness such as prosody modeling.
+- For TTV-v2, we modify some components and training process (Model size: 107M --> 278M)
+  1. Intermediate hidden size: 256 --> 384 
+  2. Loss masking for wav2vec reconstruction loss (I left out masking the loss for zero-padding sequences😥)
+  3. For long sentence generation, we finetune the model with full LibriTTS-train dataset without data filtering (Decrease the learning rate to 2e-5 with batch size of 8 per gpus)    
+
+## GAN VS Diffusion
+<details> 
+<summary> [Read More] </summary>
+We think that we could not confirm which is better yet. There are many advatanges for each model so you can utilize each model for your own purposes and each study must be actively conducted simultaneously.  
+
+### GAN (Specifically, GAN-based End-to-End Speech Synthesis Models)
+- (pros) Fast Inference Speed 
+- (pros) High-quality Audio
+- (cons) Slow Training Speed (Over 7~20 Days)
+- (cons) Lower Voice Style Transfer Performance than Diffusion Models
+- (cons) Perceptually High-quality but Over-smoothed Audio because of Information Bottleneck by the sampling from the low-dimensional Latent Variable
+   
+### Diffusion (Diffusion-based Mel-spectrogram Generation Models)
+- (pros) Fast Training Speed (within 3 Days)
+- (pros) High-quality Voice Style Transfer
+- (cons) Slow Inference Speed
+- (cons) Lower Audio quality than End-to-End Speech Synthesis Models 
+
+### (In this wors) Our Approaches for GAN-based End-to-End Speech Synthesis Models 
+- Improving Voice Style Transfer Performance in End-to-End Speech Synthesis Models for OOD (Zero-shot Voice Style Transfer for Novel Speaker)
+- Improving the Audio Quality beyond Perceptal Quality for Much more High-fidelity Audio Generation
+
+### (Our other works) Diffusion-based Mel-spectrogram Generation Models
+- DDDM-VC: Disentangled Denoising Diffusion Models for High-quality and High-diversity Speech Synthesis Models
+- Diff-hierVC: Hierarhical Diffusion-based Speech Synthesis Model with Diffusion-based Pitch Modeling
+
+### Our Goals
+- Integrating each model for High-quality, High-diversity and High-fidelity Speech Synthesis Models 
+</details> 
+
+## LLM-based Models
+We hope to compare LLM-based models for zero-shot TTS baselines. However, there is no public-available official implementation of LLM-based TTS models. Unfortunately, unofficial models have a poor performance in zero-shot TTS so we hope they will release their model for a fair comparison and reproducibility and for our speech community. THB I could not stand the inference speed almost 1,000 times slower than e2e models It takes 5 days to synthesize the full sentences of LibriTTS-test subsets. Even, the audio quality is so bad. I hope they will release their official source code soon. 
+
+In my very personal opinion, VITS is still the best TTS model I have ever seen. But, I acknowledge that LLM-based models have much powerful potential for their creative generative performance from the large-scale dataset but not now.
+
+## Limitation of our work
+- Slow training speed and Relatively large model size (Compared with VITS) --> Future work: Light-weight and Fast training pipeline and much larger model...
+- Could not generate realistic background sound --> Future work: adding audio generation part by disentangling speech and sound.
+- Could not generate a speech from a too long sentence becauase of our training setting. We see increasing max length could improve the model performance. However, we do not have GPUs with 80 GB 😢 
+  ```
+   # Data Filtering for limited computation resource. 
+    wav_min = 32
+    wav_max = 600 # 12s 
+    text_min = 1
+    text_max = 200
+  ```
+TTV v2 may reduce this issue significantly...!
+
+## Results [[Download]](https://drive.google.com/drive/folders/1xCrZQy9s5MT38RMQxKAtkoWUgxT5qYYW?usp=sharing)
+We have attached all samples from LibriTTS test-clean and test-other. 
+
+## Reference
+<details> 
+<summary> [Read More] </summary>
+ 
+### Our Previous Works
+- HierSpeech/HierSpeech-U for Hierarchical Speech Synthesis Framework: https://openreview.net/forum?id=awdyRVnfQKX
+- HierVST for Baseline Speech Backbone: https://www.isca-speech.org/archive/interspeech_2023/lee23i_interspeech.html
+- DDDM-VC: https://dddm-vc.github.io/
+- Diff-HierVC: https://diff-hiervc.github.io/
+  
+### Baseline Model
+- VITS: https://github.com/jaywalnut310/vits
+- NaturalSpeech 
+- NANSY for Audio Perturbation: https://github.com/revsic/torch-nansy
+- Speech Resynthesis: https://github.com/facebookresearch/speech-resynthesis
+  
+### Waveform Generator for High-quality Audio Generation
+- HiFi-GAN: https://github.com/jik876/hifi-gan 
+- BigVGAN for High-quality Generator: https://arxiv.org/abs/2206.04658
+- UnivNET: https://github.com/mindslab-ai/univnet
+- EnCodec: https://github.com/facebookresearch/encodec
+
+### Self-supervised Speech Model 
+- Wav2Vec 2.0: https://arxiv.org/abs/2006.11477
+- XLS-R: https://huggingface.co/facebook/wav2vec2-xls-r-300m
+- MMS: https://huggingface.co/facebook/facebook/mms-300m
+
+### Other Large Language Model based Speech Synthesis Model
+- VALL-E & VALL-E-X
+- SPEAR-TTS
+- NaturalSpeech 2
+- Make-a-Voice
+- MEGA-TTS & MEGA-TTS 2
+- UniAudio
+
+### AdaLN-zero
+- Dit: https://github.com/facebookresearch/DiT
+  
+Thanks for all nice works. 
+</details>
+
+## LICENSE
+- Code in this repo: MIT License
+- Model Weights: CC-BY-NC-4.0 license

activations.py

@@ -0,0 +1,120 @@
+# Implementation adapted from https://github.com/EdwardDixon/snake under the MIT license.
+#   LICENSE is in incl_licenses directory.
+
+import torch
+from torch import nn, sin, pow
+from torch.nn import Parameter
+
+
+class Snake(nn.Module):
+    '''
+    Implementation of a sine-based periodic activation function
+    Shape:
+        - Input: (B, C, T)
+        - Output: (B, C, T), same shape as the input
+    Parameters:
+        - alpha - trainable parameter
+    References:
+        - This activation function is from this paper by Liu Ziyin, Tilman Hartwig, Masahito Ueda:
+        https://arxiv.org/abs/2006.08195
+    Examples:
+        >>> a1 = snake(256)
+        >>> x = torch.randn(256)
+        >>> x = a1(x)
+    '''
+    def __init__(self, in_features, alpha=1.0, alpha_trainable=True, alpha_logscale=False):
+        '''
+        Initialization.
+        INPUT:
+            - in_features: shape of the input
+            - alpha: trainable parameter
+            alpha is initialized to 1 by default, higher values = higher-frequency.
+            alpha will be trained along with the rest of your model.
+        '''
+        super(Snake, self).__init__()
+        self.in_features = in_features
+
+        # initialize alpha
+        self.alpha_logscale = alpha_logscale
+        if self.alpha_logscale: # log scale alphas initialized to zeros
+            self.alpha = Parameter(torch.zeros(in_features) * alpha)
+        else: # linear scale alphas initialized to ones
+            self.alpha = Parameter(torch.ones(in_features) * alpha)
+
+        self.alpha.requires_grad = alpha_trainable
+
+        self.no_div_by_zero = 0.000000001
+
+    def forward(self, x):
+        '''
+        Forward pass of the function.
+        Applies the function to the input elementwise.
+        Snake ∶= x + 1/a * sin^2 (xa)
+        '''
+        alpha = self.alpha.unsqueeze(0).unsqueeze(-1) # line up with x to [B, C, T]
+        if self.alpha_logscale:
+            alpha = torch.exp(alpha)
+        x = x + (1.0 / (alpha + self.no_div_by_zero)) * pow(sin(x * alpha), 2)
+
+        return x
+
+
+class SnakeBeta(nn.Module):
+    '''
+    A modified Snake function which uses separate parameters for the magnitude of the periodic components
+    Shape:
+        - Input: (B, C, T)
+        - Output: (B, C, T), same shape as the input
+    Parameters:
+        - alpha - trainable parameter that controls frequency
+        - beta - trainable parameter that controls magnitude
+    References:
+        - This activation function is a modified version based on this paper by Liu Ziyin, Tilman Hartwig, Masahito Ueda:
+        https://arxiv.org/abs/2006.08195
+    Examples:
+        >>> a1 = snakebeta(256)
+        >>> x = torch.randn(256)
+        >>> x = a1(x)
+    '''
+    def __init__(self, in_features, alpha=1.0, alpha_trainable=True, alpha_logscale=False):
+        '''
+        Initialization.
+        INPUT:
+            - in_features: shape of the input
+            - alpha - trainable parameter that controls frequency
+            - beta - trainable parameter that controls magnitude
+            alpha is initialized to 1 by default, higher values = higher-frequency.
+            beta is initialized to 1 by default, higher values = higher-magnitude.
+            alpha will be trained along with the rest of your model.
+        '''
+        super(SnakeBeta, self).__init__()
+        self.in_features = in_features
+
+        # initialize alpha
+        self.alpha_logscale = alpha_logscale
+        if self.alpha_logscale: # log scale alphas initialized to zeros
+            self.alpha = Parameter(torch.zeros(in_features) * alpha)
+            self.beta = Parameter(torch.zeros(in_features) * alpha)
+        else: # linear scale alphas initialized to ones
+            self.alpha = Parameter(torch.ones(in_features) * alpha)
+            self.beta = Parameter(torch.ones(in_features) * alpha)
+
+        self.alpha.requires_grad = alpha_trainable
+        self.beta.requires_grad = alpha_trainable
+
+        self.no_div_by_zero = 0.000000001
+
+    def forward(self, x):
+        '''
+        Forward pass of the function.
+        Applies the function to the input elementwise.
+        SnakeBeta ∶= x + 1/b * sin^2 (xa)
+        '''
+        alpha = self.alpha.unsqueeze(0).unsqueeze(-1) # line up with x to [B, C, T]
+        beta = self.beta.unsqueeze(0).unsqueeze(-1)
+        if self.alpha_logscale:
+            alpha = torch.exp(alpha)
+            beta = torch.exp(beta)
+        x = x + (1.0 / (beta + self.no_div_by_zero)) * pow(sin(x * alpha), 2)
+
+        return x

alias_free_torch/__init__.py

@@ -0,0 +1,6 @@
+# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0
+#   LICENSE is in incl_licenses directory.
+
+from .filter import *
+from .resample import *
+from .act import *

alias_free_torch/act.py

@@ -0,0 +1,28 @@
+# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0
+#   LICENSE is in incl_licenses directory.
+
+import torch.nn as nn
+from .resample import UpSample1d, DownSample1d
+
+
+class Activation1d(nn.Module):
+    def __init__(self,
+                 activation,
+                 up_ratio: int = 2,
+                 down_ratio: int = 2,
+                 up_kernel_size: int = 12,
+                 down_kernel_size: int = 12):
+        super().__init__()
+        self.up_ratio = up_ratio
+        self.down_ratio = down_ratio
+        self.act = activation
+        self.upsample = UpSample1d(up_ratio, up_kernel_size)
+        self.downsample = DownSample1d(down_ratio, down_kernel_size)
+
+    # x: [B,C,T]
+    def forward(self, x):
+        x = self.upsample(x)
+        x = self.act(x)
+        x = self.downsample(x)
+
+        return x

alias_free_torch/filter.py

@@ -0,0 +1,95 @@
+# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0
+#   LICENSE is in incl_licenses directory.
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import math
+
+if 'sinc' in dir(torch):
+    sinc = torch.sinc
+else:
+    # This code is adopted from adefossez's julius.core.sinc under the MIT License
+    # https://adefossez.github.io/julius/julius/core.html
+    #   LICENSE is in incl_licenses directory.
+    def sinc(x: torch.Tensor):
+        """
+        Implementation of sinc, i.e. sin(pi * x) / (pi * x)
+        __Warning__: Different to julius.sinc, the input is multiplied by `pi`!
+        """
+        return torch.where(x == 0,
+                           torch.tensor(1., device=x.device, dtype=x.dtype),
+                           torch.sin(math.pi * x) / math.pi / x)
+
+
+# This code is adopted from adefossez's julius.lowpass.LowPassFilters under the MIT License
+# https://adefossez.github.io/julius/julius/lowpass.html
+#   LICENSE is in incl_licenses directory.
+def kaiser_sinc_filter1d(cutoff, half_width, kernel_size): # return filter [1,1,kernel_size]
+    even = (kernel_size % 2 == 0)
+    half_size = kernel_size // 2
+
+    #For kaiser window
+    delta_f = 4 * half_width
+    A = 2.285 * (half_size - 1) * math.pi * delta_f + 7.95
+    if A > 50.:
+        beta = 0.1102 * (A - 8.7)
+    elif A >= 21.:
+        beta = 0.5842 * (A - 21)**0.4 + 0.07886 * (A - 21.)
+    else:
+        beta = 0.
+    window = torch.kaiser_window(kernel_size, beta=beta, periodic=False)
+
+    # ratio = 0.5/cutoff -> 2 * cutoff = 1 / ratio
+    if even:
+        time = (torch.arange(-half_size, half_size) + 0.5)
+    else:
+        time = torch.arange(kernel_size) - half_size
+    if cutoff == 0:
+        filter_ = torch.zeros_like(time)
+    else:
+        filter_ = 2 * cutoff * window * sinc(2 * cutoff * time)
+        # Normalize filter to have sum = 1, otherwise we will have a small leakage
+        # of the constant component in the input signal.
+        filter_ /= filter_.sum()
+        filter = filter_.view(1, 1, kernel_size)
+
+    return filter
+
+
+class LowPassFilter1d(nn.Module):
+    def __init__(self,
+                 cutoff=0.5,
+                 half_width=0.6,
+                 stride: int = 1,
+                 padding: bool = True,
+                 padding_mode: str = 'replicate',
+                 kernel_size: int = 12):
+        # kernel_size should be even number for stylegan3 setup,
+        # in this implementation, odd number is also possible.
+        super().__init__()
+        if cutoff < -0.:
+            raise ValueError("Minimum cutoff must be larger than zero.")
+        if cutoff > 0.5:
+            raise ValueError("A cutoff above 0.5 does not make sense.")
+        self.kernel_size = kernel_size
+        self.even = (kernel_size % 2 == 0)
+        self.pad_left = kernel_size // 2 - int(self.even)
+        self.pad_right = kernel_size // 2
+        self.stride = stride
+        self.padding = padding
+        self.padding_mode = padding_mode
+        filter = kaiser_sinc_filter1d(cutoff, half_width, kernel_size)
+        self.register_buffer("filter", filter)
+
+    #input [B, C, T]
+    def forward(self, x):
+        _, C, _ = x.shape
+
+        if self.padding:
+            x = F.pad(x, (self.pad_left, self.pad_right),
+                      mode=self.padding_mode)
+        out = F.conv1d(x, self.filter.expand(C, -1, -1),
+                       stride=self.stride, groups=C)
+
+        return out

alias_free_torch/resample.py

@@ -0,0 +1,49 @@
+# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0
+#   LICENSE is in incl_licenses directory.
+
+import torch.nn as nn
+from torch.nn import functional as F
+from .filter import LowPassFilter1d
+from .filter import kaiser_sinc_filter1d
+
+
+class UpSample1d(nn.Module):
+    def __init__(self, ratio=2, kernel_size=None):
+        super().__init__()
+        self.ratio = ratio
+        self.kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size
+        self.stride = ratio
+        self.pad = self.kernel_size // ratio - 1
+        self.pad_left = self.pad * self.stride + (self.kernel_size - self.stride) // 2
+        self.pad_right = self.pad * self.stride + (self.kernel_size - self.stride + 1) // 2
+        filter = kaiser_sinc_filter1d(cutoff=0.5 / ratio,
+                                      half_width=0.6 / ratio,
+                                      kernel_size=self.kernel_size)
+        self.register_buffer("filter", filter)
+
+    # x: [B, C, T]
+    def forward(self, x):
+        _, C, _ = x.shape
+
+        x = F.pad(x, (self.pad, self.pad), mode='replicate')
+        x = self.ratio * F.conv_transpose1d(
+            x, self.filter.expand(C, -1, -1), stride=self.stride, groups=C)
+        x = x[..., self.pad_left:-self.pad_right]
+
+        return x
+
+
+class DownSample1d(nn.Module):
+    def __init__(self, ratio=2, kernel_size=None):
+        super().__init__()
+        self.ratio = ratio
+        self.kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size
+        self.lowpass = LowPassFilter1d(cutoff=0.5 / ratio,
+                                       half_width=0.6 / ratio,
+                                       stride=ratio,
+                                       kernel_size=self.kernel_size)
+
+    def forward(self, x):
+        xx = self.lowpass(x)
+
+        return xx

app.py.py

@@ -0,0 +1,236 @@
+import os
+import torch
+import argparse
+import numpy as np
+from scipy.io.wavfile import write
+import torchaudio
+import utils
+from Mels_preprocess import MelSpectrogramFixed
+
+from hierspeechpp_speechsynthesizer import (
+    SynthesizerTrn
+)
+from ttv_v1.text import text_to_sequence
+from ttv_v1.t2w2v_transformer import SynthesizerTrn as Text2W2V
+from speechsr24k.speechsr import SynthesizerTrn as SpeechSR24
+from speechsr48k.speechsr import SynthesizerTrn as SpeechSR48
+from denoiser.generator import MPNet
+from denoiser.infer import denoise
+
+import gradio as gr
+
+def load_text(fp):
+    with open(fp, 'r') as f:
+        filelist = [line.strip() for line in f.readlines()]
+    return filelist
+def load_checkpoint(filepath, device):
+    print(filepath)
+    assert os.path.isfile(filepath)
+    print("Loading '{}'".format(filepath))
+    checkpoint_dict = torch.load(filepath, map_location=device)
+    print("Complete.")
+    return checkpoint_dict
+def get_param_num(model):
+    num_param = sum(param.numel() for param in model.parameters())
+    return num_param
+def intersperse(lst, item):
+  result = [item] * (len(lst) * 2 + 1)
+  result[1::2] = lst
+  return result
+def add_blank_token(text):
+
+    text_norm = intersperse(text, 0)
+    text_norm = torch.LongTensor(text_norm)
+    return text_norm
+
+def tts(text, 
+        prompt, 
+        ttv_temperature, 
+        vc_temperature, 
+        duratuion_temperature, 
+        duratuion_length, 
+        denoise_ratio, 
+        random_seed):
+    
+    torch.manual_seed(random_seed)
+    torch.cuda.manual_seed(random_seed)
+    np.random.seed(random_seed)
+
+    text_len = len(text)
+    if text_len > 200:
+        raise gr.Error("Text length limited to 200 characters for this demo. Current text length is " + str(text_len))
+       
+    else:
+        text = text_to_sequence(str(text), ["english_cleaners2"])
+        
+        token = add_blank_token(text).unsqueeze(0).cuda()
+        token_length = torch.LongTensor([token.size(-1)]).cuda() 
+
+        # Prompt load
+        # sample_rate, audio = prompt
+        # audio = torch.FloatTensor([audio]).cuda()
+        # if audio.shape[0] != 1:
+        #     audio = audio[:1,:] 
+        # audio = audio / 32768 
+        audio, sample_rate = torchaudio.load(prompt)
+
+        # support only single channel
+
+        # Resampling
+        if sample_rate != 16000:
+            audio = torchaudio.functional.resample(audio, sample_rate, 16000, resampling_method="kaiser_window") 
+
+        # We utilize a hop size of 320 but denoiser uses a hop size of 400 so we utilize a hop size of 1600
+        ori_prompt_len = audio.shape[-1]
+        p = (ori_prompt_len // 1600 + 1) * 1600 - ori_prompt_len
+        audio = torch.nn.functional.pad(audio, (0, p), mode='constant').data
+
+        # If you have a memory issue during denosing the prompt, try to denoise the prompt with cpu before TTS 
+        # We will have a plan to replace a memory-efficient denoiser 
+        if denoise == 0:
+            audio = torch.cat([audio.cuda(), audio.cuda()], dim=0)
+        else:
+            with torch.no_grad():
+                
+                if ori_prompt_len > 80000:
+                    denoised_audio = []
+                    for i in range((ori_prompt_len//80000)):
+                        denoised_audio.append(denoise(audio.squeeze(0).cuda()[i*80000:(i+1)*80000], denoiser, hps_denoiser))
+                    
+                    denoised_audio.append(denoise(audio.squeeze(0).cuda()[(i+1)*80000:], denoiser, hps_denoiser))
+                    denoised_audio = torch.cat(denoised_audio, dim=1)
+                else:
+                    denoised_audio = denoise(audio.squeeze(0).cuda(), denoiser, hps_denoiser)
+
+            audio = torch.cat([audio.cuda(), denoised_audio[:,:audio.shape[-1]]], dim=0)
+
+        audio = audio[:,:ori_prompt_len]  # 20231108 We found that large size of padding decreases a performance so we remove the paddings after denosing.
+
+        if audio.shape[-1]<48000:
+            audio = torch.cat([audio,audio,audio,audio,audio], dim=1)
+
+        src_mel = mel_fn(audio.cuda())
+
+        src_length = torch.LongTensor([src_mel.size(2)]).to(device)
+        src_length2 = torch.cat([src_length,src_length], dim=0)
+
+        ## TTV (Text --> W2V, F0)
+        with torch.no_grad():
+            w2v_x, pitch = text2w2v.infer_noise_control(token, token_length, src_mel, src_length2, 
+                                                        noise_scale=ttv_temperature, noise_scale_w=duratuion_temperature, 
+                                                        length_scale=duratuion_length, denoise_ratio=denoise_ratio)
+            src_length = torch.LongTensor([w2v_x.size(2)]).cuda()  
+        
+            pitch[pitch<torch.log(torch.tensor([55]).cuda())]  = 0
+
+            ## Hierarchical Speech Synthesizer (W2V, F0 --> 16k Audio)
+            converted_audio = \
+                net_g.voice_conversion_noise_control(w2v_x, src_length, src_mel, src_length2, pitch, noise_scale=vc_temperature, denoise_ratio=denoise_ratio)
+    
+            converted_audio = speechsr(converted_audio)
+
+        converted_audio = converted_audio.squeeze()
+
+        converted_audio = converted_audio / (torch.abs(converted_audio).max()) * 32767.0 * 0.999 
+        converted_audio = converted_audio.cpu().numpy().astype('int16')
+
+        write('output.wav', 48000, converted_audio)
+        return 'output.wav'
+
+def main():
+    print('Initializing Inference Process..')
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--input_prompt', default='example/steve-jobs-2005.wav')
+    parser.add_argument('--input_txt', default='example/abstract.txt')
+    parser.add_argument('--output_dir', default='output')
+    parser.add_argument('--ckpt', default='./logs/hierspeechpp_eng_kor/hierspeechpp_v2_ckpt.pth')
+    parser.add_argument('--ckpt_text2w2v', '-ct', help='text2w2v checkpoint path', default='./logs/ttv_libritts_v1/ttv_lt960_ckpt.pth')
+    parser.add_argument('--ckpt_sr', type=str, default='./speechsr24k/G_340000.pth')  
+    parser.add_argument('--ckpt_sr48', type=str, default='./speechsr48k/G_100000.pth')  
+    parser.add_argument('--denoiser_ckpt', type=str, default='denoiser/g_best')
+    parser.add_argument('--scale_norm', type=str, default='max')
+    parser.add_argument('--output_sr', type=float, default=48000)
+    parser.add_argument('--noise_scale_ttv', type=float,
+                        default=0.333)
+    parser.add_argument('--noise_scale_vc', type=float,
+                        default=0.333)
+    parser.add_argument('--denoise_ratio', type=float,
+                        default=0.8)
+    parser.add_argument('--duration_ratio', type=float,
+                        default=0.8)
+    parser.add_argument('--seed', type=int,
+                        default=1111)
+    a = parser.parse_args()
+
+    global device, hps, hps_t2w2v,h_sr,h_sr48, hps_denoiser
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+    hps = utils.get_hparams_from_file(os.path.join(os.path.split(a.ckpt)[0], 'config.json'))
+    hps_t2w2v = utils.get_hparams_from_file(os.path.join(os.path.split(a.ckpt_text2w2v)[0], 'config.json'))
+    h_sr = utils.get_hparams_from_file(os.path.join(os.path.split(a.ckpt_sr)[0], 'config.json') )
+    h_sr48 = utils.get_hparams_from_file(os.path.join(os.path.split(a.ckpt_sr48)[0], 'config.json') )
+    hps_denoiser = utils.get_hparams_from_file(os.path.join(os.path.split(a.denoiser_ckpt)[0], 'config.json'))
+
+    global mel_fn, net_g, text2w2v, speechsr, denoiser
+
+    mel_fn = MelSpectrogramFixed(
+        sample_rate=hps.data.sampling_rate,
+        n_fft=hps.data.filter_length,
+        win_length=hps.data.win_length,
+        hop_length=hps.data.hop_length,
+        f_min=hps.data.mel_fmin,
+        f_max=hps.data.mel_fmax,
+        n_mels=hps.data.n_mel_channels,
+        window_fn=torch.hann_window
+    ).cuda()  
+
+    net_g = SynthesizerTrn(hps.data.filter_length // 2 + 1,
+        hps.train.segment_size // hps.data.hop_length,
+        **hps.model).cuda()
+    net_g.load_state_dict(torch.load(a.ckpt))
+    _ = net_g.eval()
+
+    text2w2v = Text2W2V(hps.data.filter_length // 2 + 1,
+    hps.train.segment_size // hps.data.hop_length,
+    **hps_t2w2v.model).cuda()
+    text2w2v.load_state_dict(torch.load(a.ckpt_text2w2v))
+    text2w2v.eval()
+  
+    speechsr = SpeechSR48(h_sr48.data.n_mel_channels,
+        h_sr48.train.segment_size // h_sr48.data.hop_length,
+        **h_sr48.model).cuda()
+    utils.load_checkpoint(a.ckpt_sr48, speechsr, None)
+    speechsr.eval()
+           
+    denoiser = MPNet(hps_denoiser).cuda()
+    state_dict = load_checkpoint(a.denoiser_ckpt, device)
+    denoiser.load_state_dict(state_dict['generator'])
+    denoiser.eval()
+
+    demo_play = gr.Interface(fn = tts,
+                     inputs = [gr.Textbox(max_lines=6, label="Input Text", value="HierSpeech is a zero shot speech synthesis model, which can generate high-quality audio", info="Up to 200 characters"), 
+                               gr.Audio(type='filepath', value="./example/3_rick_gt.wav"), 
+                               gr.Slider(0,1,0.333), 
+                               gr.Slider(0,1,0.333), 
+                               gr.Slider(0,1,1.0), 
+                               gr.Slider(0.5,2,1.0), 
+                               gr.Slider(0,1,0), 
+                               gr.Slider(0,9999,1111)],
+                     outputs = 'audio', 
+                     title = 'HierSpeech++',
+                     description  =  '''<div>
+                            <p style="text-align: left"> HierSpeech++ is a zero-shot speech synthesis model.</p>
+                            <p style="text-align: left"> Our model is trained with LibriTTS dataset so this model only supports english. We will release a multi-lingual HierSpeech++ soon.</p>
+                            <p style="text-align: left"> <a href="https://sh-lee-prml.github.io/HierSpeechpp-demo/">[Demo Page]</a> <a href="https://github.com/sh-lee-prml/HierSpeechpp">[Source Code]</a></p>
+                        </div>''',                      
+                     examples=[["HierSpeech is a zero-shot speech synthesis model, which can generate high-quality audio", "./example/3_rick_gt.wav", 0.333,0.333, 1.0, 1.0, 0, 1111],
+                                ["HierSpeech is a zero-shot speech synthesis model, which can generate high-quality audio", "./example/ex01_whisper_00359.wav", 0.333,0.333, 1.0, 1.0, 0, 1111],
+                               ["Hi there, I'm your new voice clone. Try your best to upload quality audio", "./example/female.wav", 0.333,0.333, 1.0, 1.0, 0, 1111],
+                               ["Hello I'm HierSpeech++", "./example/reference_1.wav", 0.333,0.333, 1.0, 1.0, 0, 1111],
+                               ]
+                    )
+    demo_play.launch(share=True, server_port=8888)
+
+if __name__ == '__main__':
+    main()

attentions.py

@@ -0,0 +1,313 @@
+import copy
+import math
+import numpy as np
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+import commons
+import modules
+from modules import LayerNorm
+
+
+class Encoder(nn.Module):
+    def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., window_size=4,
+                 **kwargs):
+        super().__init__()
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.window_size = window_size
+
+        self.drop = nn.Dropout(p_dropout)
+        self.attn_layers = nn.ModuleList()
+        self.norm_layers_1 = nn.ModuleList()
+        self.ffn_layers = nn.ModuleList()
+        self.norm_layers_2 = nn.ModuleList()
+        for i in range(self.n_layers):
+            self.attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout,
+                                                       window_size=window_size))
+            self.norm_layers_1.append(LayerNorm(hidden_channels))
+            self.ffn_layers.append(
+                FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout))
+            self.norm_layers_2.append(LayerNorm(hidden_channels))
+
+    def forward(self, x, x_mask):
+        attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
+        x = x * x_mask
+        for i in range(self.n_layers):
+            y = self.attn_layers[i](x, x, attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_1[i](x + y)
+
+            y = self.ffn_layers[i](x, x_mask)
+            y = self.drop(y)
+            x = self.norm_layers_2[i](x + y)
+        x = x * x_mask
+        return x
+
+
+class Decoder(nn.Module):
+    def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0.,
+                 proximal_bias=False, proximal_init=True, **kwargs):
+        super().__init__()
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.proximal_bias = proximal_bias
+        self.proximal_init = proximal_init
+
+        self.drop = nn.Dropout(p_dropout)
+        self.self_attn_layers = nn.ModuleList()
+        self.norm_layers_0 = nn.ModuleList()
+        self.encdec_attn_layers = nn.ModuleList()
+        self.norm_layers_1 = nn.ModuleList()
+        self.ffn_layers = nn.ModuleList()
+        self.norm_layers_2 = nn.ModuleList()
+        for i in range(self.n_layers):
+            self.self_attn_layers.append(
+                MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout,
+                                   proximal_bias=proximal_bias, proximal_init=proximal_init))
+            self.norm_layers_0.append(LayerNorm(hidden_channels))
+            self.encdec_attn_layers.append(
+                MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout))
+            self.norm_layers_1.append(LayerNorm(hidden_channels))
+            self.ffn_layers.append(
+                FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout, causal=True))
+            self.norm_layers_2.append(LayerNorm(hidden_channels))
+
+    def forward(self, x, x_mask, h, h_mask):
+        """
+        x: decoder input
+        h: encoder output
+        """
+        self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(device=x.device, dtype=x.dtype)
+        encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
+        x = x * x_mask
+        for i in range(self.n_layers):
+            y = self.self_attn_layers[i](x, x, self_attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_0[i](x + y)
+
+            y = self.encdec_attn_layers[i](x, h, encdec_attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_1[i](x + y)
+
+            y = self.ffn_layers[i](x, x_mask)
+            y = self.drop(y)
+            x = self.norm_layers_2[i](x + y)
+        x = x * x_mask
+        return x
+
+
+class MultiHeadAttention(nn.Module):
+    def __init__(self, channels, out_channels, n_heads, p_dropout=0., window_size=None, heads_share=True,
+                 block_length=None, proximal_bias=False, proximal_init=False):
+        super().__init__()
+        assert channels % n_heads == 0
+
+        self.channels = channels
+        self.out_channels = out_channels
+        self.n_heads = n_heads
+        self.p_dropout = p_dropout
+        self.window_size = window_size
+        self.heads_share = heads_share
+        self.block_length = block_length
+        self.proximal_bias = proximal_bias
+        self.proximal_init = proximal_init
+        self.attn = None
+
+        self.k_channels = channels // n_heads
+        self.conv_q = nn.Conv1d(channels, channels, 1)
+        self.conv_k = nn.Conv1d(channels, channels, 1)
+        self.conv_v = nn.Conv1d(channels, channels, 1)
+        self.conv_o = nn.Conv1d(channels, out_channels, 1)
+        self.drop = nn.Dropout(p_dropout)
+
+        if window_size is not None:
+            n_heads_rel = 1 if heads_share else n_heads
+            rel_stddev = self.k_channels ** -0.5
+            self.emb_rel_k = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev)
+            self.emb_rel_v = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev)
+
+        nn.init.xavier_uniform_(self.conv_q.weight)
+        nn.init.xavier_uniform_(self.conv_k.weight)
+        nn.init.xavier_uniform_(self.conv_v.weight)
+        if proximal_init:
+            with torch.no_grad():
+                self.conv_k.weight.copy_(self.conv_q.weight)
+                self.conv_k.bias.copy_(self.conv_q.bias)
+
+    def forward(self, x, c, attn_mask=None):
+        q = self.conv_q(x)
+        k = self.conv_k(c)
+        v = self.conv_v(c)
+
+        x, self.attn = self.attention(q, k, v, mask=attn_mask)
+
+        x = self.conv_o(x)
+        return x
+
+    def attention(self, query, key, value, mask=None):
+        # reshape [b, d, t] -> [b, n_h, t, d_k]
+        b, d, t_s, t_t = (*key.size(), query.size(2))
+        query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3)
+        key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
+        value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
+
+        scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1))
+        if self.window_size is not None:
+            assert t_s == t_t, "Relative attention is only available for self-attention."
+            key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s)
+            rel_logits = self._matmul_with_relative_keys(query / math.sqrt(self.k_channels), key_relative_embeddings)
+            scores_local = self._relative_position_to_absolute_position(rel_logits)
+            scores = scores + scores_local
+        if self.proximal_bias:
+            assert t_s == t_t, "Proximal bias is only available for self-attention."
+            scores = scores + self._attention_bias_proximal(t_s).to(device=scores.device, dtype=scores.dtype)
+        if mask is not None:
+            scores = scores.masked_fill(mask == 0, -1e4)
+            if self.block_length is not None:
+                assert t_s == t_t, "Local attention is only available for self-attention."
+                block_mask = torch.ones_like(scores).triu(-self.block_length).tril(self.block_length)
+                scores = scores.masked_fill(block_mask == 0, -1e4)
+        p_attn = F.softmax(scores, dim=-1)  # [b, n_h, t_t, t_s]
+        p_attn = self.drop(p_attn)
+        output = torch.matmul(p_attn, value)
+        if self.window_size is not None:
+            relative_weights = self._absolute_position_to_relative_position(p_attn)
+            value_relative_embeddings = self._get_relative_embeddings(self.emb_rel_v, t_s)
+            output = output + self._matmul_with_relative_values(relative_weights, value_relative_embeddings)
+        output = output.transpose(2, 3).contiguous().view(b, d, t_t)  # [b, n_h, t_t, d_k] -> [b, d, t_t]
+        return output, p_attn
+
+    def _matmul_with_relative_values(self, x, y):
+        """
+        x: [b, h, l, m]
+        y: [h or 1, m, d]
+        ret: [b, h, l, d]
+        """
+        ret = torch.matmul(x, y.unsqueeze(0))
+        return ret
+
+    def _matmul_with_relative_keys(self, x, y):
+        """
+        x: [b, h, l, d]
+        y: [h or 1, m, d]
+        ret: [b, h, l, m]
+        """
+        ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
+        return ret
+
+    def _get_relative_embeddings(self, relative_embeddings, length):
+        max_relative_position = 2 * self.window_size + 1
+        # Pad first before slice to avoid using cond ops.
+        pad_length = max(length - (self.window_size + 1), 0)
+        slice_start_position = max((self.window_size + 1) - length, 0)
+        slice_end_position = slice_start_position + 2 * length - 1
+        if pad_length > 0:
+            padded_relative_embeddings = F.pad(
+                relative_embeddings,
+                commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]))
+        else:
+            padded_relative_embeddings = relative_embeddings
+        used_relative_embeddings = padded_relative_embeddings[:, slice_start_position:slice_end_position]
+        return used_relative_embeddings
+
+    def _relative_position_to_absolute_position(self, x):
+        """
+        x: [b, h, l, 2*l-1]
+        ret: [b, h, l, l]
+        """
+        batch, heads, length, _ = x.size()
+        # Concat columns of pad to shift from relative to absolute indexing.
+        x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, 1]]))
+
+        # Concat extra elements so to add up to shape (len+1, 2*len-1).
+        x_flat = x.view([batch, heads, length * 2 * length])
+        x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [0, length - 1]]))
+
+        # Reshape and slice out the padded elements.
+        x_final = x_flat.view([batch, heads, length + 1, 2 * length - 1])[:, :, :length, length - 1:]
+        return x_final
+
+    def _absolute_position_to_relative_position(self, x):
+        """
+        x: [b, h, l, l]
+        ret: [b, h, l, 2*l-1]
+        """
+        batch, heads, length, _ = x.size()
+        # padd along column
+        x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length - 1]]))
+        x_flat = x.view([batch, heads, length ** 2 + length * (length - 1)])
+        # add 0's in the beginning that will skew the elements after reshape
+        x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
+        x_final = x_flat.view([batch, heads, length, 2 * length])[:, :, :, 1:]
+        return x_final
+
+    def _attention_bias_proximal(self, length):
+        """Bias for self-attention to encourage attention to close positions.
+        Args:
+          length: an integer scalar.
+        Returns:
+          a Tensor with shape [1, 1, length, length]
+        """
+        r = torch.arange(length, dtype=torch.float32)
+        diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1)
+        return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0)
+
+
+class FFN(nn.Module):
+    def __init__(self, in_channels, out_channels, filter_channels, kernel_size, p_dropout=0., activation=None,
+                 causal=False):
+        super().__init__()
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.filter_channels = filter_channels
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.activation = activation
+        self.causal = causal
+
+        if causal:
+            self.padding = self._causal_padding
+        else:
+            self.padding = self._same_padding
+
+        self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size)
+        self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size)
+        self.drop = nn.Dropout(p_dropout)
+
+    def forward(self, x, x_mask):
+        x = self.conv_1(self.padding(x * x_mask))
+        if self.activation == "gelu":
+            x = x * torch.sigmoid(1.702 * x)
+        else:
+            x = torch.relu(x)
+        x = self.drop(x)
+        x = self.conv_2(self.padding(x * x_mask))
+        return x * x_mask
+
+    def _causal_padding(self, x):
+        if self.kernel_size == 1:
+            return x
+        pad_l = self.kernel_size - 1
+        pad_r = 0
+        padding = [[0, 0], [0, 0], [pad_l, pad_r]]
+        x = F.pad(x, commons.convert_pad_shape(padding))
+        return x
+
+    def _same_padding(self, x):
+        if self.kernel_size == 1:
+            return x
+        pad_l = (self.kernel_size - 1) // 2
+        pad_r = self.kernel_size // 2
+        padding = [[0, 0], [0, 0], [pad_l, pad_r]]
+        x = F.pad(x, commons.convert_pad_shape(padding))
+        return x

commons.py

@@ -0,0 +1,168 @@
+import math
+import numpy as np
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+
+def init_weights(m, mean=0.0, std=0.01):
+  classname = m.__class__.__name__
+  if classname.find("Conv") != -1:
+    m.weight.data.normal_(mean, std)
+
+
+def get_padding(kernel_size, dilation=1):
+  return int((kernel_size*dilation - dilation)/2)
+
+
+def convert_pad_shape(pad_shape):
+  l = pad_shape[::-1]
+  pad_shape = [item for sublist in l for item in sublist]
+  return pad_shape
+
+
+def intersperse(lst, item):
+  result = [item] * (len(lst) * 2 + 1)
+  result[1::2] = lst
+  return result
+
+
+def kl_divergence(m_p, logs_p, m_q, logs_q):
+  """KL(P||Q)"""
+  kl = (logs_q - logs_p) - 0.5
+  kl += 0.5 * (torch.exp(2. * logs_p) + ((m_p - m_q)**2)) * torch.exp(-2. * logs_q)
+  return kl
+
+
+def rand_gumbel(shape):
+  """Sample from the Gumbel distribution, protect from overflows."""
+  uniform_samples = torch.rand(shape) * 0.99998 + 0.00001
+  return -torch.log(-torch.log(uniform_samples))
+
+
+def rand_gumbel_like(x):
+  g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device)
+  return g
+
+
+def slice_segments(x, ids_str, segment_size=4):
+  ret = torch.zeros_like(x[:, :, :segment_size])
+  for i in range(x.size(0)):
+    idx_str = ids_str[i]
+    idx_end = idx_str + segment_size
+    ret[i] = x[i, :, idx_str:idx_end]
+  return ret
+
+def slice_segments_audio(x, ids_str, segment_size=4):
+  ret = torch.zeros_like(x[:, :segment_size])
+  for i in range(x.size(0)):
+    idx_str = ids_str[i]
+    idx_end = idx_str + segment_size
+    ret[i] = x[i, idx_str:idx_end]
+  return ret
+
+def rand_slice_segments(x, x_lengths=None, segment_size=4):
+  b, d, t = x.size()
+  if x_lengths is None:
+    x_lengths = t
+  ids_str_max = x_lengths - segment_size + 1
+  ids_str = ((torch.rand([b]).to(device=x.device) * ids_str_max).clip(0)).to(dtype=torch.long)
+  ret = slice_segments(x, ids_str, segment_size)
+  return ret, ids_str
+
+
+def get_timing_signal_1d(
+    length, channels, min_timescale=1.0, max_timescale=1.0e4):
+  position = torch.arange(length, dtype=torch.float)
+  num_timescales = channels // 2
+  log_timescale_increment = (
+      math.log(float(max_timescale) / float(min_timescale)) /
+      (num_timescales - 1))
+  inv_timescales = min_timescale * torch.exp(
+      torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment)
+  scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1)
+  signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0)
+  signal = F.pad(signal, [0, 0, 0, channels % 2])
+  signal = signal.view(1, channels, length)
+  return signal
+
+
+def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4):
+  b, channels, length = x.size()
+  signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
+  return x + signal.to(dtype=x.dtype, device=x.device)
+
+
+def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1):
+  b, channels, length = x.size()
+  signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
+  return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis)
+
+
+def subsequent_mask(length):
+  mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
+  return mask
+
+
+@torch.jit.script
+def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
+  n_channels_int = n_channels[0]
+  in_act = input_a + input_b
+  t_act = torch.tanh(in_act[:, :n_channels_int, :])
+  s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
+  acts = t_act * s_act
+  return acts
+
+
+def convert_pad_shape(pad_shape):
+  l = pad_shape[::-1]
+  pad_shape = [item for sublist in l for item in sublist]
+  return pad_shape
+
+
+def shift_1d(x):
+  x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1]
+  return x
+
+
+def sequence_mask(length, max_length=None):
+  if max_length is None:
+    max_length = length.max()
+  x = torch.arange(max_length, dtype=length.dtype, device=length.device)
+  return x.unsqueeze(0) < length.unsqueeze(1)
+
+
+def generate_path(duration, mask):
+  """
+  duration: [b, 1, t_x]
+  mask: [b, 1, t_y, t_x]
+  """
+  device = duration.device
+  
+  b, _, t_y, t_x = mask.shape
+  cum_duration = torch.cumsum(duration, -1)
+  
+  cum_duration_flat = cum_duration.view(b * t_x)
+  path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
+  path = path.view(b, t_x, t_y)
+  path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1]
+  path = path.unsqueeze(1).transpose(2,3) * mask
+  return path
+
+
+def clip_grad_value_(parameters, clip_value, norm_type=2):
+  if isinstance(parameters, torch.Tensor):
+    parameters = [parameters]
+  parameters = list(filter(lambda p: p.grad is not None, parameters))
+  norm_type = float(norm_type)
+  if clip_value is not None:
+    clip_value = float(clip_value)
+
+  total_norm = 0
+  for p in parameters:
+    param_norm = p.grad.data.norm(norm_type)
+    total_norm += param_norm.item() ** norm_type
+    if clip_value is not None:
+      p.grad.data.clamp_(min=-clip_value, max=clip_value)
+  total_norm = total_norm ** (1. / norm_type)
+  return total_norm

denoiser/config.json

@@ -0,0 +1,28 @@
+{
+    "num_gpus": 0,
+    "batch_size": 4,
+    "learning_rate": 0.0005,
+    "adam_b1": 0.8,
+    "adam_b2": 0.99,
+    "lr_decay": 0.99,
+    "seed": 1234,
+    
+    "dense_channel": 64,
+    "compress_factor": 0.3, 
+    "num_tsconformers": 4,
+    "beta": 2.0,
+
+    "sampling_rate": 16000,
+    "segment_size": 32000,
+    "n_fft": 400,
+    "hop_size": 100,
+    "win_size": 400,
+
+    "num_workers": 4,
+
+    "dist_config": {
+        "dist_backend": "nccl",
+        "dist_url": "tcp://localhost:54321",
+        "world_size": 1
+    }
+}

denoiser/conformer.py

@@ -0,0 +1,86 @@
+import torch
+import torch.nn as nn
+from einops.layers.torch import Rearrange
+
+def get_padding(kernel_size, dilation=1):
+    return int((kernel_size*dilation - dilation)/2)
+
+class FeedForwardModule(nn.Module):
+    def __init__(self, dim, mult=4, dropout=0):
+        super(FeedForwardModule, self).__init__()
+        self.ffm = nn.Sequential(
+            nn.LayerNorm(dim),
+            nn.Linear(dim, dim * mult),
+            nn.SiLU(),
+            nn.Dropout(dropout),
+            nn.Linear(dim * mult, dim),
+            nn.Dropout(dropout)
+        )
+
+    def forward(self, x):
+        return self.ffm(x)
+
+
+class ConformerConvModule(nn.Module):
+    def __init__(self, dim, expansion_factor=2, kernel_size=31, dropout=0.):
+        super(ConformerConvModule, self).__init__()
+        inner_dim = dim * expansion_factor
+        self.ccm = nn.Sequential(
+            nn.LayerNorm(dim),
+            Rearrange('b n c -> b c n'),
+            nn.Conv1d(dim, inner_dim*2, 1),
+            nn.GLU(dim=1),
+            nn.Conv1d(inner_dim, inner_dim, kernel_size=kernel_size,
+                      padding=get_padding(kernel_size), groups=inner_dim), # DepthWiseConv1d 
+            nn.BatchNorm1d(inner_dim),
+            nn.SiLU(),
+            nn.Conv1d(inner_dim, dim, 1),
+            Rearrange('b c n -> b n c'),
+            nn.Dropout(dropout)
+        )
+
+    def forward(self, x):
+        return self.ccm(x)
+
+
+class AttentionModule(nn.Module):
+    def __init__(self, dim, n_head=8, dropout=0.):
+        super(AttentionModule, self).__init__()
+        self.attn = nn.MultiheadAttention(dim, n_head, dropout=dropout)
+        self.layernorm = nn.LayerNorm(dim)
+
+    def forward(self, x, attn_mask=None, key_padding_mask=None):
+        x = self.layernorm(x)
+        x, _ = self.attn(x, x, x, 
+                         attn_mask=attn_mask,
+                         key_padding_mask=key_padding_mask)
+        return x
+
+
+class ConformerBlock(nn.Module):
+    def __init__(self, dim, n_head=8, ffm_mult=4, ccm_expansion_factor=2, ccm_kernel_size=31, 
+                 ffm_dropout=0., attn_dropout=0., ccm_dropout=0.):
+        super(ConformerBlock, self).__init__()
+        self.ffm1 = FeedForwardModule(dim, ffm_mult, dropout=ffm_dropout)
+        self.attn = AttentionModule(dim, n_head, dropout=attn_dropout)
+        self.ccm = ConformerConvModule(dim, ccm_expansion_factor, ccm_kernel_size, dropout=ccm_dropout)
+        self.ffm2 = FeedForwardModule(dim, ffm_mult, dropout=ffm_dropout)
+        self.post_norm = nn.LayerNorm(dim)
+
+    def forward(self, x):
+        x = x + 0.5 * self.ffm1(x)
+        x = x + self.attn(x)
+        x = x + self.ccm(x)
+        x = x + 0.5 * self.ffm2(x)
+        x = self.post_norm(x)
+        return x
+
+
+def main():
+    x = torch.ones(10, 100, 64)
+    conformer = ConformerBlock(dim=64)
+    print(conformer(x))
+
+
+if __name__ == '__main__':
+    main()

denoiser/g_best

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0088cd06068a97b97cc13fe10fe155ea5c24beea79564b2162fab22a79dc9dc5
+size 8350488

denoiser/generator.py

@@ -0,0 +1,193 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+from denoiser.conformer import ConformerBlock
+from denoiser.utils import get_padding_2d, LearnableSigmoid_2d
+from pesq import pesq
+from joblib import Parallel, delayed
+
+class DenseBlock(nn.Module):
+    def __init__(self, h, kernel_size=(3, 3), depth=4):
+        super(DenseBlock, self).__init__()
+        self.h = h
+        self.depth = depth
+        self.dense_block = nn.ModuleList([])
+        for i in range(depth):
+            dil = 2 ** i
+            dense_conv = nn.Sequential(
+                nn.Conv2d(h.dense_channel*(i+1), h.dense_channel, kernel_size, dilation=(dil, 1),
+                          padding=get_padding_2d(kernel_size, (dil, 1))),
+                nn.InstanceNorm2d(h.dense_channel, affine=True),
+                nn.PReLU(h.dense_channel)
+            )
+            self.dense_block.append(dense_conv)
+
+    def forward(self, x):
+        skip = x
+        for i in range(self.depth):
+            x = self.dense_block[i](skip)
+            skip = torch.cat([x, skip], dim=1)
+        return x
+
+
+class DenseEncoder(nn.Module):
+    def __init__(self, h, in_channel):
+        super(DenseEncoder, self).__init__()
+        self.h = h
+        self.dense_conv_1 = nn.Sequential(
+            nn.Conv2d(in_channel, h.dense_channel, (1, 1)),
+            nn.InstanceNorm2d(h.dense_channel, affine=True),
+            nn.PReLU(h.dense_channel))
+
+        self.dense_block = DenseBlock(h, depth=4) # [b, h.dense_channel, ndim_time, h.n_fft//2+1]
+
+        self.dense_conv_2 = nn.Sequential(
+            nn.Conv2d(h.dense_channel, h.dense_channel, (1, 3), (1, 2)),
+            nn.InstanceNorm2d(h.dense_channel, affine=True),
+            nn.PReLU(h.dense_channel))
+
+    def forward(self, x):
+        x = self.dense_conv_1(x)  # [b, 64, T, F]
+        x = self.dense_block(x)   # [b, 64, T, F]
+        x = self.dense_conv_2(x)  # [b, 64, T, F//2]
+        return x
+
+
+class MaskDecoder(nn.Module):
+    def __init__(self, h, out_channel=1):
+        super(MaskDecoder, self).__init__()
+        self.dense_block = DenseBlock(h, depth=4)
+        self.mask_conv = nn.Sequential(
+            nn.ConvTranspose2d(h.dense_channel, h.dense_channel, (1, 3), (1, 2)),
+            nn.Conv2d(h.dense_channel, out_channel, (1, 1)),
+            nn.InstanceNorm2d(out_channel, affine=True),
+            nn.PReLU(out_channel),
+            nn.Conv2d(out_channel, out_channel, (1, 1))
+        )
+        self.lsigmoid = LearnableSigmoid_2d(h.n_fft//2+1, beta=h.beta)
+
+    def forward(self, x):
+        x = self.dense_block(x)
+        x = self.mask_conv(x)
+        x = x.permute(0, 3, 2, 1).squeeze(-1)
+        x = self.lsigmoid(x).permute(0, 2, 1).unsqueeze(1)
+        return x
+
+
+class PhaseDecoder(nn.Module):
+    def __init__(self, h, out_channel=1):
+        super(PhaseDecoder, self).__init__()
+        self.dense_block = DenseBlock(h, depth=4)
+        self.phase_conv = nn.Sequential(
+            nn.ConvTranspose2d(h.dense_channel, h.dense_channel, (1, 3), (1, 2)),
+            nn.InstanceNorm2d(h.dense_channel, affine=True),
+            nn.PReLU(h.dense_channel)
+        )
+        self.phase_conv_r = nn.Conv2d(h.dense_channel, out_channel, (1, 1))
+        self.phase_conv_i = nn.Conv2d(h.dense_channel, out_channel, (1, 1))
+
+    def forward(self, x):
+        x = self.dense_block(x)
+        x = self.phase_conv(x)
+        x_r = self.phase_conv_r(x)
+        x_i = self.phase_conv_i(x)
+        x = torch.atan2(x_i, x_r)
+        return x
+
+
+class TSConformerBlock(nn.Module):
+    def __init__(self, h):
+        super(TSConformerBlock, self).__init__()
+        self.h = h
+        self.time_conformer = ConformerBlock(dim=h.dense_channel,  n_head=4, ccm_kernel_size=31, 
+                                             ffm_dropout=0.2, attn_dropout=0.2)
+        self.freq_conformer = ConformerBlock(dim=h.dense_channel,  n_head=4, ccm_kernel_size=31, 
+                                             ffm_dropout=0.2, attn_dropout=0.2)
+
+    def forward(self, x):
+        b, c, t, f = x.size()
+        x = x.permute(0, 3, 2, 1).contiguous().view(b*f, t, c)
+        x = self.time_conformer(x) + x
+        x = x.view(b, f, t, c).permute(0, 2, 1, 3).contiguous().view(b*t, f, c)
+        x = self.freq_conformer(x) + x
+        x = x.view(b, t, f, c).permute(0, 3, 1, 2)
+        return x
+
+
+class MPNet(nn.Module):
+    def __init__(self, h, num_tscblocks=4):
+        super(MPNet, self).__init__()
+        self.h = h
+        self.num_tscblocks = num_tscblocks
+        self.dense_encoder = DenseEncoder(h, in_channel=2)
+
+        self.TSConformer = nn.ModuleList([])
+        for i in range(num_tscblocks):
+            self.TSConformer.append(TSConformerBlock(h))
+        
+        self.mask_decoder = MaskDecoder(h, out_channel=1)
+        self.phase_decoder = PhaseDecoder(h, out_channel=1)
+
+    def forward(self, noisy_mag, noisy_pha): # [B, F, T]
+        noisy_mag = noisy_mag.unsqueeze(-1).permute(0, 3, 2, 1) # [B, 1, T, F]
+        noisy_pha = noisy_pha.unsqueeze(-1).permute(0, 3, 2, 1) # [B, 1, T, F]
+        x = torch.cat((noisy_mag, noisy_pha), dim=1) # [B, 2, T, F]
+        x = self.dense_encoder(x)
+
+        for i in range(self.num_tscblocks):
+            x = self.TSConformer[i](x)
+        
+        denoised_mag = (noisy_mag * self.mask_decoder(x)).permute(0, 3, 2, 1).squeeze(-1)
+        denoised_pha = self.phase_decoder(x).permute(0, 3, 2, 1).squeeze(-1)
+        denoised_com = torch.stack((denoised_mag*torch.cos(denoised_pha),
+                                    denoised_mag*torch.sin(denoised_pha)), dim=-1)
+
+        return denoised_mag, denoised_pha, denoised_com
+
+
+def phase_losses(phase_r, phase_g, h):
+
+    dim_freq = h.n_fft // 2 + 1
+    dim_time = phase_r.size(-1)
+
+    gd_matrix = (torch.triu(torch.ones(dim_freq, dim_freq), diagonal=1) - torch.triu(torch.ones(dim_freq, dim_freq), diagonal=2) - torch.eye(dim_freq)).to(phase_g.device)
+    gd_r = torch.matmul(phase_r.permute(0, 2, 1), gd_matrix)
+    gd_g = torch.matmul(phase_g.permute(0, 2, 1), gd_matrix)
+
+    iaf_matrix = (torch.triu(torch.ones(dim_time, dim_time), diagonal=1) - torch.triu(torch.ones(dim_time, dim_time), diagonal=2) - torch.eye(dim_time)).to(phase_g.device)
+    iaf_r = torch.matmul(phase_r, iaf_matrix)
+    iaf_g = torch.matmul(phase_g, iaf_matrix)
+
+    ip_loss = torch.mean(anti_wrapping_function(phase_r-phase_g))
+    gd_loss = torch.mean(anti_wrapping_function(gd_r-gd_g))
+    iaf_loss = torch.mean(anti_wrapping_function(iaf_r-iaf_g))
+
+    return ip_loss, gd_loss, iaf_loss
+
+
+def anti_wrapping_function(x):
+
+    return torch.abs(x - torch.round(x / (2 * np.pi)) * 2 * np.pi)
+
+
+def pesq_score(utts_r, utts_g, h):
+
+    pesq_score = Parallel(n_jobs=30)(delayed(eval_pesq)(
+                            utts_r[i].squeeze().cpu().numpy(),
+                            utts_g[i].squeeze().cpu().numpy(), 
+                            h.sampling_rate)
+                          for i in range(len(utts_r)))
+    pesq_score = np.mean(pesq_score)
+
+    return pesq_score
+
+
+def eval_pesq(clean_utt, esti_utt, sr):
+    try:
+        pesq_score = pesq(sr, clean_utt, esti_utt)
+    except:
+        # error can happen due to silent period
+        pesq_score = -1
+
+    return pesq_score

denoiser/infer.py

@@ -0,0 +1,33 @@
+
+import torch
+
+def denoise(noisy_wav, model, hps):
+    norm_factor = torch.sqrt(len(noisy_wav) / torch.sum(noisy_wav ** 2.0)).to(noisy_wav.device)
+    noisy_wav = (noisy_wav * norm_factor).unsqueeze(0)
+    noisy_amp, noisy_pha, noisy_com = mag_pha_stft(noisy_wav, hps.n_fft, hps.hop_size, hps.win_size, hps.compress_factor)
+    amp_g, pha_g, com_g = model(noisy_amp, noisy_pha)
+    audio_g = mag_pha_istft(amp_g, pha_g, hps.n_fft, hps.hop_size, hps.win_size, hps.compress_factor)
+    audio_g = audio_g / norm_factor
+    return audio_g
+
+def mag_pha_stft(y, n_fft, hop_size, win_size, compress_factor=1.0, center=True):
+
+    hann_window = torch.hann_window(win_size).to(y.device)
+    stft_spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window,
+                           center=center, pad_mode='reflect', normalized=False, return_complex=True)
+    mag = torch.abs(stft_spec)
+    pha = torch.angle(stft_spec)
+    # Magnitude Compression
+    mag = torch.pow(mag, compress_factor)
+    com = torch.stack((mag*torch.cos(pha), mag*torch.sin(pha)), dim=-1)
+
+    return mag, pha, com
+
+def mag_pha_istft(mag, pha, n_fft, hop_size, win_size, compress_factor=1.0, center=True):
+    # Magnitude Decompression
+    mag = torch.pow(mag, (1.0/compress_factor))
+    com = torch.complex(mag*torch.cos(pha), mag*torch.sin(pha))
+    hann_window = torch.hann_window(win_size).to(com.device)
+    wav = torch.istft(com, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window, center=center)
+
+    return wav

denoiser/utils.py

@@ -0,0 +1,55 @@
+import glob
+import os
+import torch
+import torch.nn as nn
+
+def get_padding(kernel_size, dilation=1):
+    return int((kernel_size*dilation - dilation)/2)
+
+
+def get_padding_2d(kernel_size, dilation=(1, 1)):
+    return (int((kernel_size[0]*dilation[0] - dilation[0])/2), int((kernel_size[1]*dilation[1] - dilation[1])/2))
+
+
+def load_checkpoint(filepath, device):
+    assert os.path.isfile(filepath)
+    print("Loading '{}'".format(filepath))
+    checkpoint_dict = torch.load(filepath, map_location=device)
+    print("Complete.")
+    return checkpoint_dict
+
+
+def save_checkpoint(filepath, obj):
+    print("Saving checkpoint to {}".format(filepath))
+    torch.save(obj, filepath)
+    print("Complete.")
+
+
+def scan_checkpoint(cp_dir, prefix):
+    pattern = os.path.join(cp_dir, prefix + '????????')
+    cp_list = glob.glob(pattern)
+    if len(cp_list) == 0:
+        return None
+    return sorted(cp_list)[-1]
+
+
+class LearnableSigmoid_1d(nn.Module):
+    def __init__(self, in_features, beta=1):
+        super().__init__()
+        self.beta = beta
+        self.slope = nn.Parameter(torch.ones(in_features))
+        self.slope.requiresGrad = True
+
+    def forward(self, x):
+        return self.beta * torch.sigmoid(self.slope * x)
+
+
+class LearnableSigmoid_2d(nn.Module):
+    def __init__(self, in_features, beta=1):
+        super().__init__()
+        self.beta = beta
+        self.slope = nn.Parameter(torch.ones(in_features, 1))
+        self.slope.requiresGrad = True
+
+    def forward(self, x):
+        return self.beta * torch.sigmoid(self.slope * x)

example/reference_1.txt

@@ -0,0 +1 @@
+And lay me down in my cold bed and leave my shining lot.

example/reference_2.txt

@@ -0,0 +1 @@
+Yea, his honourable worship is within, but he hath a godly minister or two with him, and likewise a leech.

example/reference_3.txt

@@ -0,0 +1 @@
+The army found the people in poverty and left them in comparative wealth.

example/reference_4.txt

@@ -0,0 +1 @@
+Thus did this humane and right minded father comfort his unhappy daughter, and her mother embracing her again, did all she could to soothe her feelings.

hierspeechpp_speechsynthesizer.py

@@ -0,0 +1,716 @@
+import torch
+from torch import nn
+from torch.nn import functional as F
+import modules
+import attentions
+
+from torch.nn import Conv1d, ConvTranspose1d, Conv2d
+from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
+from commons import init_weights, get_padding
+
+import torchaudio
+from einops import rearrange
+import transformers
+import math
+from styleencoder import StyleEncoder
+import commons
+
+from alias_free_torch import *
+import activations
+
+class Wav2vec2(torch.nn.Module):
+    def __init__(self, layer=7, w2v='mms'):
+
+        """we use the intermediate features of mms-300m.
+           More specifically, we used the output from the 7th layer of the 24-layer transformer encoder.
+        """
+        super().__init__()
+
+        if w2v == 'mms':
+           self.wav2vec2 = transformers.Wav2Vec2ForPreTraining.from_pretrained("facebook/mms-300m")
+        else:
+           self.wav2vec2 = transformers.Wav2Vec2ForPreTraining.from_pretrained("facebook/wav2vec2-xls-r-300m")
+
+        for param in self.wav2vec2.parameters():
+            param.requires_grad = False
+            param.grad = None
+        self.wav2vec2.eval()
+        self.feature_layer = layer
+
+    @torch.no_grad()
+    def forward(self, x):
+        """
+        Args:
+            x: torch.Tensor of shape (B x t)
+        Returns:
+            y: torch.Tensor of shape(B x C x t)
+        """
+        outputs = self.wav2vec2(x.squeeze(1), output_hidden_states=True)
+        y = outputs.hidden_states[self.feature_layer]  # B x t x C(1024)
+        y = y.permute((0, 2, 1))  # B x t x C -> B x C x t
+        return y
+
+class ResidualCouplingBlock_Transformer(nn.Module):
+  def __init__(self,
+      channels,
+      hidden_channels,
+      kernel_size,
+      dilation_rate,
+      n_layers=3,
+      n_flows=4,
+      gin_channels=0):
+    super().__init__()
+    self.channels = channels
+    self.hidden_channels = hidden_channels
+    self.kernel_size = kernel_size
+    self.dilation_rate = dilation_rate
+    self.n_layers = n_layers
+    self.n_flows = n_flows
+    self.gin_channels = gin_channels
+    self.cond_block = torch.nn.Sequential(torch.nn.Linear(gin_channels, 4 * hidden_channels),
+                                            nn.SiLU(), torch.nn.Linear(4 * hidden_channels, hidden_channels))
+
+    self.flows = nn.ModuleList()
+    for i in range(n_flows):
+      self.flows.append(modules.ResidualCouplingLayer_Transformer_simple(channels, hidden_channels, kernel_size, dilation_rate, n_layers, mean_only=True))
+      self.flows.append(modules.Flip())
+
+  def forward(self, x, x_mask, g=None, reverse=False):
+
+    g = self.cond_block(g.squeeze(2))
+
+    if not reverse:
+      for flow in self.flows:
+        x, _ = flow(x, x_mask, g=g, reverse=reverse)
+    else:
+      for flow in reversed(self.flows):
+        x = flow(x, x_mask, g=g, reverse=reverse)
+    return x
+
+class PosteriorAudioEncoder(nn.Module):
+  def __init__(self,
+      in_channels,
+      out_channels,
+      hidden_channels,
+      kernel_size,
+      dilation_rate,
+      n_layers,
+      gin_channels=0):
+    super().__init__()
+    self.in_channels = in_channels
+    self.out_channels = out_channels
+    self.hidden_channels = hidden_channels
+    self.kernel_size = kernel_size
+    self.dilation_rate = dilation_rate
+    self.n_layers = n_layers
+    self.gin_channels = gin_channels
+
+    self.pre = nn.Conv1d(in_channels, hidden_channels, 1)
+    self.down_pre = nn.Conv1d(1, 16, 7, 1, padding=3)
+    self.resblocks = nn.ModuleList()
+    downsample_rates = [8,5,4,2]
+    downsample_kernel_sizes = [17, 10, 8, 4]
+    ch = [16, 32, 64, 128, 192]
+
+    resblock = AMPBlock1
+    resblock_kernel_sizes = [3,7,11]
+    resblock_dilation_sizes = [[1,3,5], [1,3,5], [1,3,5]]
+    self.num_kernels = 3
+    self.downs = nn.ModuleList()
+    for i, (u, k) in enumerate(zip(downsample_rates, downsample_kernel_sizes)):
+        self.downs.append(weight_norm(
+            Conv1d(ch[i], ch[i+1], k, u, padding=(k-1)//2)))
+    for i in range(4):
+        for j, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)):
+            self.resblocks.append(resblock(ch[i+1], k, d, activation="snakebeta"))
+
+    activation_post = activations.SnakeBeta(ch[i+1], alpha_logscale=True)
+    self.activation_post = Activation1d(activation=activation_post)
+
+    self.conv_post = Conv1d(ch[i+1], hidden_channels, 7, 1, padding=3)
+
+
+    self.enc = modules.WN(hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels)
+    self.proj = nn.Conv1d(hidden_channels*2, out_channels * 2, 1)
+
+  def forward(self, x, x_audio, x_mask, g=None):
+
+    x_audio = self.down_pre(x_audio)
+
+    for i in range(4):
+
+        x_audio = self.downs[i](x_audio)
+
+        xs = None
+        for j in range(self.num_kernels):
+            if xs is None:
+                xs = self.resblocks[i*self.num_kernels+j](x_audio)
+            else:
+                xs += self.resblocks[i*self.num_kernels+j](x_audio)
+        x_audio = xs / self.num_kernels
+
+    x_audio = self.activation_post(x_audio)
+    x_audio = self.conv_post(x_audio)
+
+    x = self.pre(x) * x_mask
+    x = self.enc(x, x_mask, g=g)
+
+    x_audio = x_audio * x_mask
+
+    x = torch.cat([x, x_audio], dim=1)
+
+    stats = self.proj(x) * x_mask
+
+    m, logs = torch.split(stats, self.out_channels, dim=1)
+    z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask
+    return z, m, logs
+
+class PosteriorSFEncoder(nn.Module):
+  def __init__(self,
+      src_channels,
+      out_channels,
+      hidden_channels,
+      kernel_size,
+      dilation_rate,
+      n_layers,
+      gin_channels=0):
+    super().__init__()
+
+    self.out_channels = out_channels
+    self.hidden_channels = hidden_channels
+    self.kernel_size = kernel_size
+    self.dilation_rate = dilation_rate
+    self.n_layers = n_layers
+    self.gin_channels = gin_channels
+
+    self.pre_source = nn.Conv1d(src_channels, hidden_channels, 1)
+    self.pre_filter = nn.Conv1d(1, hidden_channels, kernel_size=9, stride=4, padding=4)
+    self.source_enc = modules.WN(hidden_channels, kernel_size, dilation_rate, n_layers//2, gin_channels=gin_channels)
+    self.filter_enc = modules.WN(hidden_channels, kernel_size, dilation_rate, n_layers//2, gin_channels=gin_channels)
+    self.enc = modules.WN(hidden_channels, kernel_size, dilation_rate, n_layers//2, gin_channels=gin_channels)
+    self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
+
+  def forward(self, x_src, x_ftr, x_mask, g=None):
+
+    x_src = self.pre_source(x_src) * x_mask
+    x_ftr = self.pre_filter(x_ftr) * x_mask
+    x_src = self.source_enc(x_src, x_mask, g=g)
+    x_ftr = self.filter_enc(x_ftr, x_mask, g=g)
+    x = self.enc(x_src+x_ftr, x_mask, g=g)
+    stats = self.proj(x) * x_mask
+    m, logs = torch.split(stats, self.out_channels, dim=1)
+    z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask
+    return z, m, logs
+
+
+class MelDecoder(nn.Module):
+  def __init__(self,
+      hidden_channels,
+      filter_channels,
+      n_heads,
+      n_layers,
+      kernel_size,
+      p_dropout,
+      mel_size=20,
+      gin_channels=0):
+    super().__init__()
+
+    self.hidden_channels = hidden_channels
+    self.filter_channels = filter_channels
+    self.n_heads = n_heads
+    self.n_layers = n_layers
+    self.kernel_size = kernel_size
+    self.p_dropout = p_dropout
+
+    self.conv_pre = Conv1d(hidden_channels, hidden_channels, 3, 1, padding=1)
+
+    self.encoder = attentions.Encoder(
+      hidden_channels,
+      filter_channels,
+      n_heads,
+      n_layers,
+      kernel_size,
+      p_dropout)
+
+    self.proj= nn.Conv1d(hidden_channels, mel_size, 1, bias=False)
+
+    if gin_channels != 0:
+      self.cond = nn.Conv1d(gin_channels, hidden_channels, 1)
+
+  def forward(self, x, x_mask, g=None):
+
+    x = self.conv_pre(x*x_mask)
+    if g is not None:
+        x = x + self.cond(g)
+
+    x = self.encoder(x * x_mask, x_mask)
+    x = self.proj(x) * x_mask
+
+    return x
+
+class SourceNetwork(nn.Module):
+  def __init__(self, upsample_initial_channel=256):
+    super().__init__()
+
+    resblock_kernel_sizes = [3,5,7]
+    upsample_rates = [2,2]
+    initial_channel = 192
+    upsample_initial_channel = upsample_initial_channel
+    upsample_kernel_sizes = [4,4]
+    resblock_dilation_sizes = [[1,3,5], [1,3,5], [1,3,5]]
+
+    self.num_kernels = len(resblock_kernel_sizes)
+    self.num_upsamples = len(upsample_rates)
+
+    self.conv_pre = weight_norm(Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3))
+    resblock = AMPBlock1
+
+    self.ups = nn.ModuleList()
+    for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
+        self.ups.append(weight_norm(
+            ConvTranspose1d(upsample_initial_channel//(2**i), upsample_initial_channel//(2**(i+1)),
+                            k, u, padding=(k-u)//2)))
+
+    self.resblocks = nn.ModuleList()
+    for i in range(len(self.ups)):
+        ch = upsample_initial_channel//(2**(i+1))
+        for j, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)):
+            self.resblocks.append(resblock(ch, k, d, activation="snakebeta"))
+
+    activation_post = activations.SnakeBeta(ch, alpha_logscale=True)
+    self.activation_post = Activation1d(activation=activation_post)
+
+    self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
+
+    self.cond = Conv1d(256, upsample_initial_channel, 1)
+
+    self.ups.apply(init_weights)
+
+
+  def forward(self, x,  g):
+
+    x = self.conv_pre(x) + self.cond(g)
+
+    for i in range(self.num_upsamples):
+
+      x = self.ups[i](x)
+      xs = None
+      for j in range(self.num_kernels):
+        if xs is None:
+          xs = self.resblocks[i*self.num_kernels+j](x)
+        else:
+          xs += self.resblocks[i*self.num_kernels+j](x)
+      x = xs / self.num_kernels
+
+    x = self.activation_post(x)
+    ## Predictor
+    x_ = self.conv_post(x)
+    return x, x_
+  
+def remove_weight_norm(self):
+    print('Removing weight norm...')
+    for l in self.ups:
+        remove_weight_norm(l)
+    for l in self.resblocks:
+        l.remove_weight_norm()
+
+class DBlock(nn.Module):
+  def __init__(self, input_size, hidden_size, factor):
+    super().__init__()
+    self.factor = factor
+    self.residual_dense = weight_norm(Conv1d(input_size, hidden_size, 1))
+    self.conv = nn.ModuleList([
+        weight_norm(Conv1d(input_size, hidden_size, 3, dilation=1, padding=1)),
+        weight_norm(Conv1d(hidden_size, hidden_size, 3, dilation=2, padding=2)),
+        weight_norm(Conv1d(hidden_size, hidden_size, 3, dilation=4, padding=4)),
+    ])
+    self.conv.apply(init_weights)
+  def forward(self, x):
+    size = x.shape[-1] // self.factor
+
+    residual = self.residual_dense(x)
+    residual = F.interpolate(residual, size=size)
+
+    x = F.interpolate(x, size=size)
+    for layer in self.conv:
+      x = F.leaky_relu(x, modules.LRELU_SLOPE)
+      x = layer(x)
+
+    return x + residual
+  def remove_weight_norm(self):
+    for l in self.conv:
+        remove_weight_norm(l)
+
+class AMPBlock1(torch.nn.Module):
+    def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5), activation=None):
+        super(AMPBlock1, self).__init__()
+      
+        self.convs1 = nn.ModuleList([
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
+                               padding=get_padding(kernel_size, dilation[0]))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
+                               padding=get_padding(kernel_size, dilation[1]))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2],
+                               padding=get_padding(kernel_size, dilation[2])))
+        ])
+        self.convs1.apply(init_weights)
+
+        self.convs2 = nn.ModuleList([
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+                               padding=get_padding(kernel_size, 1))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+                               padding=get_padding(kernel_size, 1))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+                               padding=get_padding(kernel_size, 1)))
+        ])
+        self.convs2.apply(init_weights)
+
+        self.num_layers = len(self.convs1) + len(self.convs2) # total number of conv layers
+
+
+        self.activations = nn.ModuleList([
+            Activation1d(
+                activation=activations.SnakeBeta(channels, alpha_logscale=True))
+                for _ in range(self.num_layers)
+        ])
+  
+    def forward(self, x):
+        acts1, acts2 = self.activations[::2], self.activations[1::2]
+        for c1, c2, a1, a2 in zip(self.convs1, self.convs2, acts1, acts2):
+            xt = a1(x)
+            xt = c1(xt)
+            xt = a2(xt)
+            xt = c2(xt)
+            x = xt + x
+
+        return x
+
+    def remove_weight_norm(self):
+        for l in self.convs1:
+            remove_weight_norm(l)
+        for l in self.convs2:
+            remove_weight_norm(l)
+
+class Generator(torch.nn.Module):
+    def __init__(self, initial_channel,  resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gin_channels=256):
+        super(Generator, self).__init__()
+        self.num_kernels = len(resblock_kernel_sizes)
+        self.num_upsamples = len(upsample_rates)
+
+          
+        self.conv_pre = weight_norm(Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3))
+        resblock = AMPBlock1
+
+        self.ups = nn.ModuleList()
+        for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
+            self.ups.append(weight_norm(
+                ConvTranspose1d(upsample_initial_channel//(2**i), upsample_initial_channel//(2**(i+1)),
+                                k, u, padding=(k-u)//2)))
+
+        self.resblocks = nn.ModuleList()
+        for i in range(len(self.ups)):
+            ch = upsample_initial_channel//(2**(i+1))
+            for j, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)):
+                self.resblocks.append(resblock(ch, k, d, activation="snakebeta"))
+
+        activation_post = activations.SnakeBeta(ch, alpha_logscale=True)
+        self.activation_post = Activation1d(activation=activation_post)
+
+        self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
+        self.ups.apply(init_weights)
+
+        if gin_channels != 0:
+            self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)
+
+        self.downs = DBlock(upsample_initial_channel//8, upsample_initial_channel, 4)
+        self.proj = Conv1d(upsample_initial_channel//8, upsample_initial_channel//2, 7, 1, padding=3)
+
+    def forward(self, x, pitch, g=None):
+
+        x = self.conv_pre(x) + self.downs(pitch) + self.cond(g)
+
+        for i in range(self.num_upsamples):
+
+            x = self.ups[i](x)
+
+            if i == 0:
+                pitch = self.proj(pitch)
+                x = x + pitch
+
+            xs = None
+            for j in range(self.num_kernels):
+                if xs is None:
+                    xs = self.resblocks[i*self.num_kernels+j](x)
+                else:
+                    xs += self.resblocks[i*self.num_kernels+j](x)
+            x = xs / self.num_kernels
+
+        x = self.activation_post(x)
+        x = self.conv_post(x)
+        x = torch.tanh(x)
+        return x
+
+    def remove_weight_norm(self):
+        print('Removing weight norm...')
+        for l in self.ups:
+            remove_weight_norm(l)
+        for l in self.resblocks:
+            l.remove_weight_norm()
+        for l in self.downs:
+            l.remove_weight_norm()
+        remove_weight_norm(self.conv_pre)
+  
+class DiscriminatorP(torch.nn.Module):
+    def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
+        super(DiscriminatorP, self).__init__()
+        self.period = period
+        self.use_spectral_norm = use_spectral_norm
+        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
+        self.convs = nn.ModuleList([
+            norm_f(Conv2d(1, 32, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(Conv2d(32, 128, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(Conv2d(128, 512, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(Conv2d(512, 1024, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(Conv2d(1024, 1024, (kernel_size, 1), 1, padding=(get_padding(kernel_size, 1), 0))),
+        ])
+        self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
+
+    def forward(self, x):
+        fmap = []
+
+        # 1d to 2d
+        b, c, t = x.shape
+        if t % self.period != 0: # pad first
+            n_pad = self.period - (t % self.period)
+            x = F.pad(x, (0, n_pad), "reflect")
+            t = t + n_pad
+        x = x.view(b, c, t // self.period, self.period)
+
+        for l in self.convs:
+            x = l(x)
+            x = F.leaky_relu(x, modules.LRELU_SLOPE)
+            fmap.append(x)
+        x = self.conv_post(x)
+        fmap.append(x)
+        x = torch.flatten(x, 1, -1)
+
+        return x, fmap
+
+class DiscriminatorR(torch.nn.Module):
+    def __init__(self, resolution, use_spectral_norm=False):
+        super(DiscriminatorR, self).__init__()
+        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
+
+        n_fft, hop_length, win_length = resolution
+        self.spec_transform = torchaudio.transforms.Spectrogram(
+            n_fft=n_fft, hop_length=hop_length, win_length=win_length, window_fn=torch.hann_window,
+            normalized=True, center=False, pad_mode=None, power=None)
+
+        self.convs = nn.ModuleList([
+            norm_f(nn.Conv2d(2, 32, (3, 9), padding=(1, 4))),
+            norm_f(nn.Conv2d(32, 32, (3, 9), stride=(1, 2), padding=(1, 4))),
+            norm_f(nn.Conv2d(32, 32, (3, 9), stride=(1, 2), dilation=(2,1), padding=(2, 4))),
+            norm_f(nn.Conv2d(32, 32, (3, 9), stride=(1, 2), dilation=(4,1), padding=(4, 4))),
+            norm_f(nn.Conv2d(32, 32, (3, 3), padding=(1, 1))),
+        ])
+        self.conv_post = norm_f(nn.Conv2d(32, 1, (3, 3), padding=(1, 1)))
+
+    def forward(self, y):
+        fmap = []
+
+        x = self.spec_transform(y)  # [B, 2, Freq, Frames, 2]
+        x = torch.cat([x.real, x.imag], dim=1)
+        x = rearrange(x, 'b c w t -> b c t w')
+
+        for l in self.convs:
+            x = l(x)
+            x = F.leaky_relu(x, modules.LRELU_SLOPE)
+            fmap.append(x)
+        x = self.conv_post(x)
+        fmap.append(x)
+        x = torch.flatten(x, 1, -1)
+
+        return x, fmap
+
+
+class MultiPeriodDiscriminator(torch.nn.Module):
+    def __init__(self, use_spectral_norm=False):
+        super(MultiPeriodDiscriminator, self).__init__()
+        periods = [2,3,5,7,11]
+        resolutions = [[2048, 512, 2048], [1024, 256, 1024], [512, 128, 512], [256, 64, 256], [128, 32, 128]]
+
+        discs = [DiscriminatorR(resolutions[i], use_spectral_norm=use_spectral_norm) for i in range(len(resolutions))]
+        discs = discs + [DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods]
+
+        self.discriminators = nn.ModuleList(discs)
+
+    def forward(self, y, y_hat):
+        y_d_rs = []
+        y_d_gs = []
+        fmap_rs = []
+        fmap_gs = []
+        for i, d in enumerate(self.discriminators):
+            y_d_r, fmap_r = d(y)
+            y_d_g, fmap_g = d(y_hat)
+            y_d_rs.append(y_d_r)
+            y_d_gs.append(y_d_g)
+            fmap_rs.append(fmap_r)
+            fmap_gs.append(fmap_g)
+
+        return y_d_rs, y_d_gs, fmap_rs, fmap_gs
+
+class SynthesizerTrn(nn.Module):
+  """
+  Synthesizer for Training
+  """
+
+  def __init__(self,
+
+    spec_channels,
+    segment_size,
+    inter_channels,
+    hidden_channels,
+    filter_channels,
+    n_heads,
+    n_layers,
+    kernel_size,
+    p_dropout,
+    resblock,
+    resblock_kernel_sizes,
+    resblock_dilation_sizes,
+    upsample_rates,
+    upsample_initial_channel,
+    upsample_kernel_sizes,
+    gin_channels=256,
+    prosody_size=20,
+    uncond_ratio=0.,
+    cfg=False,
+    **kwargs):
+
+    super().__init__()
+    self.spec_channels = spec_channels
+    self.inter_channels = inter_channels
+    self.hidden_channels = hidden_channels
+    self.filter_channels = filter_channels
+    self.n_heads = n_heads
+    self.n_layers = n_layers
+    self.kernel_size = kernel_size
+    self.p_dropout = p_dropout
+    self.resblock = resblock
+    self.resblock_kernel_sizes = resblock_kernel_sizes
+    self.resblock_dilation_sizes = resblock_dilation_sizes
+    self.upsample_rates = upsample_rates
+    self.upsample_initial_channel = upsample_initial_channel
+    self.upsample_kernel_sizes = upsample_kernel_sizes
+    self.segment_size = segment_size
+    self.mel_size = prosody_size
+
+    self.enc_p_l = PosteriorSFEncoder(1024, inter_channels, hidden_channels, 5, 1, 16, gin_channels=gin_channels)
+    self.flow_l = ResidualCouplingBlock_Transformer(inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels)
+
+    self.enc_p = PosteriorSFEncoder(1024, inter_channels, hidden_channels, 5, 1, 16, gin_channels=gin_channels)
+    self.enc_q = PosteriorAudioEncoder(spec_channels, inter_channels, hidden_channels, 5, 1, 16, gin_channels=gin_channels)
+    self.flow = ResidualCouplingBlock_Transformer(inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels)
+
+    self.mel_decoder = MelDecoder(inter_channels,
+                                     filter_channels,
+                                     n_heads=2,
+                                     n_layers=2,
+                                     kernel_size=5,
+                                     p_dropout=0.1,
+                                     mel_size=self.mel_size,
+                                     gin_channels=gin_channels)
+
+    self.dec = Generator(inter_channels, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gin_channels=gin_channels)
+    self.sn = SourceNetwork(upsample_initial_channel//2)
+    self.emb_g = StyleEncoder(in_dim=80, hidden_dim=256, out_dim=gin_channels)
+
+    if cfg:
+
+      self.emb = torch.nn.Embedding(1, 256)
+      torch.nn.init.normal_(self.emb.weight, 0.0, 256 ** -0.5)
+      self.null = torch.LongTensor([0]).cuda()
+      self.uncond_ratio = uncond_ratio
+    self.cfg = cfg
+  @torch.no_grad()
+  def infer(self, x_mel, w2v, length, f0):
+
+    x_mask = torch.unsqueeze(commons.sequence_mask(length, x_mel.size(2)), 1).to(x_mel.dtype)
+
+    # Speaker embedding from mel (Style Encoder)
+    g = self.emb_g(x_mel, x_mask).unsqueeze(-1)
+    
+    z, _, _ = self.enc_p_l(w2v, f0, x_mask, g=g)
+
+    z = self.flow_l(z, x_mask, g=g, reverse=True)
+    z = self.flow(z, x_mask, g=g, reverse=True)
+
+    e, e_ = self.sn(z, g)
+    o = self.dec(z, e, g=g)
+
+    return o, e_
+  @torch.no_grad()
+  def voice_conversion(self, src, src_length, trg_mel, trg_length, f0, noise_scale = 0.333, uncond=False):
+
+    trg_mask = torch.unsqueeze(commons.sequence_mask(trg_length, trg_mel.size(2)), 1).to(trg_mel.dtype)
+    g = self.emb_g(trg_mel, trg_mask).unsqueeze(-1)
+
+    y_mask = torch.unsqueeze(commons.sequence_mask(src_length, src.size(2)), 1).to(trg_mel.dtype)
+    z, m_p, logs_p = self.enc_p_l(src, f0, y_mask, g=g)
+
+    z = (m_p + torch.randn_like(m_p) * torch.exp(logs_p)*noise_scale) * y_mask
+
+    z = self.flow_l(z, y_mask, g=g, reverse=True)
+    z = self.flow(z, y_mask, g=g, reverse=True)
+
+    if uncond:
+        null_emb = self.emb(self.null) * math.sqrt(256)
+        g = null_emb.unsqueeze(-1)
+
+    e, _ = self.sn(z, g)
+    o = self.dec(z, e, g=g)
+
+    return o
+  @torch.no_grad()
+  def voice_conversion_noise_control(self, src, src_length, trg_mel, trg_length, f0, noise_scale = 0.333, uncond=False, denoise_ratio = 0):
+
+    trg_mask = torch.unsqueeze(commons.sequence_mask(trg_length, trg_mel.size(2)), 1).to(trg_mel.dtype)
+    g = self.emb_g(trg_mel, trg_mask).unsqueeze(-1)
+
+    g_org, g_denoise = g[:1, :, :], g[1:, :, :]
+
+    g_interpolation = (1-denoise_ratio)*g_org + denoise_ratio*g_denoise
+
+    y_mask = torch.unsqueeze(commons.sequence_mask(src_length, src.size(2)), 1).to(trg_mel.dtype)
+    z, m_p, logs_p = self.enc_p_l(src, f0, y_mask, g=g_interpolation)
+
+    z = (m_p + torch.randn_like(m_p) * torch.exp(logs_p)*noise_scale) * y_mask
+
+    z = self.flow_l(z, y_mask, g=g_interpolation, reverse=True)
+    z = self.flow(z, y_mask, g=g_interpolation, reverse=True)
+
+    if uncond:
+        null_emb = self.emb(self.null) * math.sqrt(256)
+        g = null_emb.unsqueeze(-1)
+
+    e, _ = self.sn(z, g_interpolation)
+    o = self.dec(z, e, g=g_interpolation)
+
+    return o
+  @torch.no_grad()  
+  def f0_extraction(self, x_linear, x_mel, length, x_audio, noise_scale = 0.333):
+
+    x_mask = torch.unsqueeze(commons.sequence_mask(length, x_mel.size(2)), 1).to(x_mel.dtype)
+
+    # Speaker embedding from mel (Style Encoder)
+    g = self.emb_g(x_mel, x_mask).unsqueeze(-1)
+
+    # posterior encoder from linear spec. 
+    _, m_q, logs_q= self.enc_q(x_linear, x_audio, x_mask, g=g)
+    z = (m_q + torch.randn_like(m_q) * torch.exp(logs_q)*noise_scale)
+
+    # Source Networks
+    _, e_ = self.sn(z, g)
+
+    return e_
+

inference.py

@@ -0,0 +1,220 @@
+import os
+import torch
+import argparse
+import numpy as np
+from scipy.io.wavfile import write
+import torchaudio
+import utils
+from Mels_preprocess import MelSpectrogramFixed
+
+from hierspeechpp_speechsynthesizer import (
+    SynthesizerTrn
+)
+from ttv_v1.text import text_to_sequence
+from ttv_v1.t2w2v_transformer import SynthesizerTrn as Text2W2V
+from speechsr24k.speechsr import SynthesizerTrn as AudioSR
+from speechsr48k.speechsr import SynthesizerTrn as AudioSR48
+from denoiser.generator import MPNet
+from denoiser.infer import denoise
+
+seed = 1111
+torch.manual_seed(seed)
+torch.cuda.manual_seed(seed)
+np.random.seed(seed)
+
+def load_text(fp):
+    with open(fp, 'r') as f:
+        filelist = [line.strip() for line in f.readlines()]
+    return filelist
+def load_checkpoint(filepath, device):
+    print(filepath)
+    assert os.path.isfile(filepath)
+    print("Loading '{}'".format(filepath))
+    checkpoint_dict = torch.load(filepath, map_location=device)
+    print("Complete.")
+    return checkpoint_dict
+def get_param_num(model):
+    num_param = sum(param.numel() for param in model.parameters())
+    return num_param
+def intersperse(lst, item):
+  result = [item] * (len(lst) * 2 + 1)
+  result[1::2] = lst
+  return result
+
+def add_blank_token(text):
+
+    text_norm = intersperse(text, 0)
+    text_norm = torch.LongTensor(text_norm)
+    return text_norm
+
+def tts(text, a, hierspeech):
+    
+    net_g, text2w2v, audiosr, denoiser, mel_fn = hierspeech
+
+    os.makedirs(a.output_dir, exist_ok=True)
+    text = text_to_sequence(str(text), ["english_cleaners2"])
+    token = add_blank_token(text).unsqueeze(0).cuda()
+    token_length = torch.LongTensor([token.size(-1)]).cuda() 
+
+    # Prompt load
+    audio, sample_rate = torchaudio.load(a.input_prompt)
+
+    # support only single channel
+    audio = audio[:1,:] 
+    # Resampling
+    if sample_rate != 16000:
+        audio = torchaudio.functional.resample(audio, sample_rate, 16000, resampling_method="kaiser_window") 
+    if a.scale_norm == 'prompt':
+        prompt_audio_max = torch.max(audio.abs())
+
+    # We utilize a hop size of 320 but denoiser uses a hop size of 400 so we utilize a hop size of 1600
+    ori_prompt_len = audio.shape[-1]
+    p = (ori_prompt_len // 1600 + 1) * 1600 - ori_prompt_len
+    audio = torch.nn.functional.pad(audio, (0, p), mode='constant').data
+
+    file_name = os.path.splitext(os.path.basename(a.input_prompt))[0]
+
+    # If you have a memory issue during denosing the prompt, try to denoise the prompt with cpu before TTS 
+    # We will have a plan to replace a memory-efficient denoiser 
+    if a.denoise_ratio == 0:
+        audio = torch.cat([audio.cuda(), audio.cuda()], dim=0)
+    else:
+        with torch.no_grad():
+            denoised_audio = denoise(audio.squeeze(0).cuda(), denoiser, hps_denoiser)
+        audio = torch.cat([audio.cuda(), denoised_audio[:,:audio.shape[-1]]], dim=0)
+
+    
+    audio = audio[:,:ori_prompt_len]  # 20231108 We found that large size of padding decreases a performance so we remove the paddings after denosing.
+
+    src_mel = mel_fn(audio.cuda())
+
+    src_length = torch.LongTensor([src_mel.size(2)]).to(device)
+    src_length2 = torch.cat([src_length,src_length], dim=0)
+
+    ## TTV (Text --> W2V, F0)
+    with torch.no_grad():
+        w2v_x, pitch = text2w2v.infer_noise_control(token, token_length, src_mel, src_length2, noise_scale=a.noise_scale_ttv, denoise_ratio=a.denoise_ratio)
+   
+        src_length = torch.LongTensor([w2v_x.size(2)]).cuda()  
+        
+        ## Pitch Clipping
+        pitch[pitch<torch.log(torch.tensor([55]).cuda())]  = 0
+
+        ## Hierarchical Speech Synthesizer (W2V, F0 --> 16k Audio)
+        converted_audio = \
+            net_g.voice_conversion_noise_control(w2v_x, src_length, src_mel, src_length2, pitch, noise_scale=a.noise_scale_vc, denoise_ratio=a.denoise_ratio)
+                
+        ## SpeechSR (Optional) (16k Audio --> 24k or 48k Audio)
+        if a.output_sr == 48000 or 24000:
+            converted_audio = audiosr(converted_audio)
+
+    converted_audio = converted_audio.squeeze()
+    
+    if a.scale_norm == 'prompt':
+        converted_audio = converted_audio / (torch.abs(converted_audio).max()) * 32767.0 * prompt_audio_max
+    else:
+        converted_audio = converted_audio / (torch.abs(converted_audio).max()) * 32767.0 * 0.999 
+
+    converted_audio = converted_audio.cpu().numpy().astype('int16')
+
+    file_name2 = "{}.wav".format(file_name)
+    output_file = os.path.join(a.output_dir, file_name2)
+    
+    if a.output_sr == 48000:
+        write(output_file, 48000, converted_audio)
+    elif a.output_sr == 24000:
+        write(output_file, 24000, converted_audio)
+    else:
+        write(output_file, 16000, converted_audio)
+
+def model_load(a):
+    mel_fn = MelSpectrogramFixed(
+        sample_rate=hps.data.sampling_rate,
+        n_fft=hps.data.filter_length,
+        win_length=hps.data.win_length,
+        hop_length=hps.data.hop_length,
+        f_min=hps.data.mel_fmin,
+        f_max=hps.data.mel_fmax,
+        n_mels=hps.data.n_mel_channels,
+        window_fn=torch.hann_window
+    ).cuda()
+    
+    net_g = SynthesizerTrn(hps.data.filter_length // 2 + 1,
+        hps.train.segment_size // hps.data.hop_length,
+        **hps.model).cuda()
+    net_g.load_state_dict(torch.load(a.ckpt))
+    _ = net_g.eval()
+
+    text2w2v = Text2W2V(hps.data.filter_length // 2 + 1,
+    hps.train.segment_size // hps.data.hop_length,
+    **hps_t2w2v.model).cuda()
+    text2w2v.load_state_dict(torch.load(a.ckpt_text2w2v))
+    text2w2v.eval()
+
+    if a.output_sr == 48000:
+        audiosr = AudioSR48(h_sr48.data.n_mel_channels,
+            h_sr48.train.segment_size // h_sr48.data.hop_length,
+            **h_sr48.model).cuda()
+        utils.load_checkpoint(a.ckpt_sr48, audiosr, None)
+        audiosr.eval()
+       
+    elif a.output_sr == 24000:
+        audiosr = AudioSR(h_sr.data.n_mel_channels,
+        h_sr.train.segment_size // h_sr.data.hop_length,
+        **h_sr.model).cuda()
+        utils.load_checkpoint(a.ckpt_sr, audiosr, None)
+        audiosr.eval()
+      
+    else:
+        audiosr = None
+    
+    denoiser = MPNet(hps_denoiser).cuda()
+    state_dict = load_checkpoint(a.denoiser_ckpt, device)
+    denoiser.load_state_dict(state_dict['generator'])
+    denoiser.eval()
+    return net_g, text2w2v, audiosr, denoiser, mel_fn
+
+def inference(a):
+    
+    hierspeech = model_load(a) 
+    # Input Text 
+    text = load_text(a.input_txt)
+    # text = "hello I'm hierspeech"
+    
+    tts(text, a, hierspeech)
+
+def main():
+    print('Initializing Inference Process..')
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--input_prompt', default='example/reference_4.wav')
+    parser.add_argument('--input_txt', default='example/reference_4.txt')
+    parser.add_argument('--output_dir', default='output')
+    parser.add_argument('--ckpt', default='./logs/hierspeechpp_eng_kor/hierspeechpp_v2_ckpt.pth')
+    parser.add_argument('--ckpt_text2w2v', '-ct', help='text2w2v checkpoint path', default='./logs/ttv_libritts_v1/ttv_lt960_ckpt.pth')
+    parser.add_argument('--ckpt_sr', type=str, default='./speechsr24k/G_340000.pth')  
+    parser.add_argument('--ckpt_sr48', type=str, default='./speechsr48k/G_100000.pth')  
+    parser.add_argument('--denoiser_ckpt', type=str, default='denoiser/g_best')
+    parser.add_argument('--scale_norm', type=str, default='max')
+    parser.add_argument('--output_sr', type=float, default=48000)
+    parser.add_argument('--noise_scale_ttv', type=float,
+                        default=0.333)
+    parser.add_argument('--noise_scale_vc', type=float,
+                        default=0.333)
+    parser.add_argument('--denoise_ratio', type=float,
+                        default=0.8)
+    a = parser.parse_args()
+
+    global device, hps, hps_t2w2v,h_sr,h_sr48, hps_denoiser
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+    hps = utils.get_hparams_from_file(os.path.join(os.path.split(a.ckpt)[0], 'config.json'))
+    hps_t2w2v = utils.get_hparams_from_file(os.path.join(os.path.split(a.ckpt_text2w2v)[0], 'config.json'))
+    h_sr = utils.get_hparams_from_file(os.path.join(os.path.split(a.ckpt_sr)[0], 'config.json') )
+    h_sr48 = utils.get_hparams_from_file(os.path.join(os.path.split(a.ckpt_sr48)[0], 'config.json') )
+    hps_denoiser = utils.get_hparams_from_file(os.path.join(os.path.split(a.denoiser_ckpt)[0], 'config.json'))
+
+    inference(a)
+
+if __name__ == '__main__':
+    main()

inference.sh

@@ -0,0 +1,12 @@
+# --ckpt "logs/hierspeechpp_libritts460/hierspeechpp_lt460_ckpt.pth" \ LibriTTS-460
+# --ckpt "logs/hierspeechpp_libritts960/hierspeechpp_lt960_ckpt.pth" \ LibriTTS-960
+# --ckpt "logs/hierspeechpp_eng_kor/hierspeechpp_v1_ckpt.pth" \ Large_v1 epoch 60 (paper version)
+# --ckpt "logs/hierspeechpp_eng_kor/hierspeechpp_v2_ckpt.pth" \ Large_v2 epoch 110 (08. Nov. 2023)
+
+CUDA_VISIBLE_DEVICES=0 python3 inference.py \
+                --ckpt "logs/hierspeechpp_eng_kor/hierspeechpp_v2_ckpt.pth" \
+                --ckpt_text2w2v "logs/ttv_libritts_v1/ttv_lt960_ckpt.pth" \
+                --output_dir "tts_results_eng_kor_v2" \
+                --noise_scale_vc "0.333" \
+                --noise_scale_ttv "0.333" \
+                --denoise_ratio "0"

inference_speechsr.py

@@ -0,0 +1,94 @@
+import os
+import torch
+import argparse
+import numpy as np
+from scipy.io.wavfile import write
+import torchaudio
+import utils
+
+from speechsr24k.speechsr import SynthesizerTrn as SpeechSR24
+from speechsr48k.speechsr import SynthesizerTrn as SpeechSR48
+
+seed = 1111
+torch.manual_seed(seed)
+torch.cuda.manual_seed(seed)
+np.random.seed(seed)
+
+def get_param_num(model):
+    num_param = sum(param.numel() for param in model.parameters())
+    return num_param
+
+def SuperResoltuion(a, hierspeech):
+    
+    speechsr = hierspeech
+
+    os.makedirs(a.output_dir, exist_ok=True)
+
+    # Prompt load
+    audio, sample_rate = torchaudio.load(a.input_speech)
+
+    # support only single channel
+    audio = audio[:1,:] 
+    # Resampling
+    if sample_rate != 16000:
+        audio = torchaudio.functional.resample(audio, sample_rate, 16000, resampling_method="kaiser_window") 
+    file_name = os.path.splitext(os.path.basename(a.input_speech))[0]
+        ## SpeechSR (Optional) (16k Audio --> 24k or 48k Audio)
+    with torch.no_grad():
+        converted_audio = speechsr(audio.unsqueeze(1).cuda())
+        converted_audio = converted_audio.squeeze()
+        converted_audio = converted_audio / (torch.abs(converted_audio).max()) * 0.999 * 32767.0
+        converted_audio = converted_audio.cpu().numpy().astype('int16')
+
+    file_name2 = "{}.wav".format(file_name)
+    output_file = os.path.join(a.output_dir, file_name2)
+    
+    if a.output_sr == 48000:
+        write(output_file, 48000, converted_audio)
+    else:
+        write(output_file, 24000, converted_audio)
+
+
+def model_load(a):
+    if a.output_sr == 48000:
+        speechsr = SpeechSR48(h_sr48.data.n_mel_channels,
+            h_sr48.train.segment_size // h_sr48.data.hop_length,
+            **h_sr48.model).cuda()
+        utils.load_checkpoint(a.ckpt_sr48, speechsr, None)
+        speechsr.eval()
+    else:
+        # 24000 Hz
+        speechsr = SpeechSR24(h_sr.data.n_mel_channels,
+        h_sr.train.segment_size // h_sr.data.hop_length,
+        **h_sr.model).cuda()
+        utils.load_checkpoint(a.ckpt_sr, speechsr, None)
+        speechsr.eval()
+    return speechsr
+
+def inference(a):
+    
+    speechsr = model_load(a) 
+    SuperResoltuion(a, speechsr)
+
+def main():
+    print('Initializing Inference Process..')
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--input_speech', default='example/reference_4.wav')
+    parser.add_argument('--output_dir', default='SR_results')
+    parser.add_argument('--ckpt_sr', type=str, default='./speechsr24k/G_340000.pth')  
+    parser.add_argument('--ckpt_sr48', type=str, default='./speechsr48k/G_100000.pth')  
+    parser.add_argument('--output_sr', type=float, default=48000)
+    a = parser.parse_args()
+
+    global device, h_sr, h_sr48
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+    h_sr = utils.get_hparams_from_file(os.path.join(os.path.split(a.ckpt_sr)[0], 'config.json') )
+    h_sr48 = utils.get_hparams_from_file(os.path.join(os.path.split(a.ckpt_sr48)[0], 'config.json') )
+
+
+    inference(a)
+
+if __name__ == '__main__':
+    main()

inference_vc.py

@@ -0,0 +1,250 @@
+import os
+import torch
+import argparse
+import numpy as np
+from scipy.io.wavfile import write
+import torchaudio
+import utils
+from Mels_preprocess import MelSpectrogramFixed
+from torch.nn import functional as F
+from hierspeechpp_speechsynthesizer import (
+    SynthesizerTrn, Wav2vec2
+)
+from ttv_v1.text import text_to_sequence
+from ttv_v1.t2w2v_transformer import SynthesizerTrn as Text2W2V
+from speechsr24k.speechsr import SynthesizerTrn as SpeechSR24
+from speechsr48k.speechsr import SynthesizerTrn as SpeechSR48
+from denoiser.generator import MPNet
+from denoiser.infer import denoise
+
+import amfm_decompy.basic_tools as basic
+import amfm_decompy.pYAAPT as pYAAPT
+
+seed = 1111
+torch.manual_seed(seed)
+torch.cuda.manual_seed(seed)
+np.random.seed(seed)
+
+def get_yaapt_f0(audio, rate=16000, interp=False):
+    frame_length = 20.0
+    to_pad = int(frame_length / 1000 * rate) // 2
+
+    f0s = []
+    for y in audio.astype(np.float64):
+        y_pad = np.pad(y.squeeze(), (to_pad, to_pad), "constant", constant_values=0)
+        signal = basic.SignalObj(y_pad, rate)
+        pitch = pYAAPT.yaapt(signal, **{'frame_length': frame_length, 'frame_space': 5.0, 'nccf_thresh1': 0.25,
+                                        'tda_frame_length': 25.0, 'f0_max':1100})
+        if interp:
+            f0s += [pitch.samp_interp[None, None, :]]
+        else:
+            f0s += [pitch.samp_values[None, None, :]] 
+    f0 = np.vstack(f0s)
+    return f0
+
+def load_text(fp):
+    with open(fp, 'r') as f:
+        filelist = [line.strip() for line in f.readlines()]
+    return filelist
+def load_checkpoint(filepath, device):
+    print(filepath)
+    assert os.path.isfile(filepath)
+    print("Loading '{}'".format(filepath))
+    checkpoint_dict = torch.load(filepath, map_location=device)
+    print("Complete.")
+    return checkpoint_dict
+def get_param_num(model):
+    num_param = sum(param.numel() for param in model.parameters())
+    return num_param
+def intersperse(lst, item):
+  result = [item] * (len(lst) * 2 + 1)
+  result[1::2] = lst
+  return result
+
+def add_blank_token(text):
+
+    text_norm = intersperse(text, 0)
+    text_norm = torch.LongTensor(text_norm)
+    return text_norm
+
+def VC(a, hierspeech):
+    
+    net_g, speechsr, denoiser, mel_fn, w2v = hierspeech
+
+    os.makedirs(a.output_dir, exist_ok=True)
+   
+    source_audio, sample_rate = torchaudio.load(a.source_speech)
+    if sample_rate != 16000:
+        source_audio = torchaudio.functional.resample(source_audio, sample_rate, 16000, resampling_method="kaiser_window")
+    p = (source_audio.shape[-1] // 1280 + 1) * 1280 - source_audio.shape[-1]
+    source_audio = torch.nn.functional.pad(source_audio, (0, p), mode='constant').data
+    file_name_s = os.path.splitext(os.path.basename(a.source_speech))[0]
+
+    try:
+        f0 = get_yaapt_f0(source_audio.numpy())
+    except:
+        f0 = np.zeros((1, 1, source_audio.shape[-1] // 80))
+        f0 = f0.astype(np.float32)
+        f0 = f0.squeeze(0) 
+
+    ii = f0 != 0
+    f0[ii] = (f0[ii] - f0[ii].mean()) / f0[ii].std()
+
+    y_pad = F.pad(source_audio, (40, 40), "reflect")
+    x_w2v = w2v(y_pad.cuda())
+    x_length = torch.LongTensor([x_w2v.size(2)]).to(device)
+
+    # Prompt load
+    target_audio, sample_rate = torchaudio.load(a.target_speech)
+    # support only single channel
+    target_audio = target_audio[:1,:] 
+    # Resampling
+    if sample_rate != 16000:
+        target_audio = torchaudio.functional.resample(target_audio, sample_rate, 16000, resampling_method="kaiser_window") 
+    if a.scale_norm == 'prompt':
+        prompt_audio_max = torch.max(target_audio.abs())
+    try:
+        t_f0 = get_yaapt_f0(target_audio.numpy())
+    except:
+        t_f0 = np.zeros((1, 1, target_audio.shape[-1] // 80))
+        t_f0 = t_f0.astype(np.float32)
+        t_f0 = t_f0.squeeze(0)
+    j = t_f0 != 0
+
+    f0[ii] = ((f0[ii] * t_f0[j].std()) + t_f0[j].mean()).clip(min=0)
+    denorm_f0 = torch.log(torch.FloatTensor(f0+1).cuda())
+    # We utilize a hop size of 320 but denoiser uses a hop size of 400 so we utilize a hop size of 1600
+    ori_prompt_len = target_audio.shape[-1]
+    p = (ori_prompt_len // 1600 + 1) * 1600 - ori_prompt_len
+    target_audio = torch.nn.functional.pad(target_audio, (0, p), mode='constant').data
+
+    file_name_t = os.path.splitext(os.path.basename(a.target_speech))[0]
+
+    # If you have a memory issue during denosing the prompt, try to denoise the prompt with cpu before TTS 
+    # We will have a plan to replace a memory-efficient denoiser 
+    if a.denoise_ratio == 0:
+        target_audio = torch.cat([target_audio.cuda(), target_audio.cuda()], dim=0)
+    else:
+        with torch.no_grad():
+            denoised_audio = denoise(target_audio.squeeze(0).cuda(), denoiser, hps_denoiser)
+        target_audio = torch.cat([target_audio.cuda(), denoised_audio[:,:target_audio.shape[-1]]], dim=0)
+
+    
+    target_audio = target_audio[:,:ori_prompt_len]  # 20231108 We found that large size of padding decreases a performance so we remove the paddings after denosing.
+
+    trg_mel = mel_fn(target_audio.cuda())
+
+    trg_length = torch.LongTensor([trg_mel.size(2)]).to(device)
+    trg_length2 = torch.cat([trg_length,trg_length], dim=0)
+
+
+    with torch.no_grad():
+        
+        ## Hierarchical Speech Synthesizer (W2V, F0 --> 16k Audio)
+        converted_audio = \
+            net_g.voice_conversion_noise_control(x_w2v, x_length, trg_mel, trg_length2, denorm_f0, noise_scale=a.noise_scale_vc, denoise_ratio=a.denoise_ratio)
+                
+        ## SpeechSR (Optional) (16k Audio --> 24k or 48k Audio)
+        if a.output_sr == 48000 or 24000:
+            converted_audio = speechsr(converted_audio)
+
+    converted_audio = converted_audio.squeeze()
+    
+    if a.scale_norm == 'prompt':
+        converted_audio = converted_audio / (torch.abs(converted_audio).max()) * 32767.0 * prompt_audio_max
+    else:
+        converted_audio = converted_audio / (torch.abs(converted_audio).max()) * 32767.0 * 0.999 
+
+    converted_audio = converted_audio.cpu().numpy().astype('int16')
+
+    file_name2 = "{}.wav".format(file_name_s+"_to_"+file_name_t)
+    output_file = os.path.join(a.output_dir, file_name2)
+    
+    if a.output_sr == 48000:
+        write(output_file, 48000, converted_audio)
+    elif a.output_sr == 24000:
+        write(output_file, 24000, converted_audio)
+    else:
+        write(output_file, 16000, converted_audio)
+
+def model_load(a):
+    mel_fn = MelSpectrogramFixed(
+        sample_rate=hps.data.sampling_rate,
+        n_fft=hps.data.filter_length,
+        win_length=hps.data.win_length,
+        hop_length=hps.data.hop_length,
+        f_min=hps.data.mel_fmin,
+        f_max=hps.data.mel_fmax,
+        n_mels=hps.data.n_mel_channels,
+        window_fn=torch.hann_window
+    ).cuda()
+    w2v = Wav2vec2().cuda()
+
+    net_g = SynthesizerTrn(hps.data.filter_length // 2 + 1,
+        hps.train.segment_size // hps.data.hop_length,
+        **hps.model).cuda()
+    net_g.load_state_dict(torch.load(a.ckpt))
+    _ = net_g.eval()
+
+    if a.output_sr == 48000:
+        speechsr = SpeechSR48(h_sr48.data.n_mel_channels,
+            h_sr48.train.segment_size // h_sr48.data.hop_length,
+            **h_sr48.model).cuda()
+        utils.load_checkpoint(a.ckpt_sr48, speechsr, None)
+        speechsr.eval()
+       
+    elif a.output_sr == 24000:
+        speechsr = SpeechSR24(h_sr.data.n_mel_channels,
+        h_sr.train.segment_size // h_sr.data.hop_length,
+        **h_sr.model).cuda()
+        utils.load_checkpoint(a.ckpt_sr, speechsr, None)
+        speechsr.eval()
+      
+    else:
+        speechsr = None
+    
+    denoiser = MPNet(hps_denoiser).cuda()
+    state_dict = load_checkpoint(a.denoiser_ckpt, device)
+    denoiser.load_state_dict(state_dict['generator'])
+    denoiser.eval()
+    return net_g, speechsr, denoiser, mel_fn, w2v
+
+def inference(a):
+    
+    hierspeech = model_load(a) 
+
+    VC(a, hierspeech)
+
+def main():
+    print('Initializing Inference Process..')
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--source_speech', default='example/reference_2.wav')
+    parser.add_argument('--target_speech', default='example/reference_1.wav')
+    parser.add_argument('--output_dir', default='output')
+    parser.add_argument('--ckpt', default='./logs/hierspeechpp_eng_kor/hierspeechpp_v2_ckpt.pth')
+    parser.add_argument('--ckpt_sr', type=str, default='./speechsr24k/G_340000.pth')  
+    parser.add_argument('--ckpt_sr48', type=str, default='./speechsr48k/G_100000.pth')  
+    parser.add_argument('--denoiser_ckpt', type=str, default='denoiser/g_best')
+    parser.add_argument('--scale_norm', type=str, default='max')
+    parser.add_argument('--output_sr', type=float, default=48000)
+    parser.add_argument('--noise_scale_ttv', type=float,
+                        default=0.333)
+    parser.add_argument('--noise_scale_vc', type=float,
+                        default=0.333)
+    parser.add_argument('--denoise_ratio', type=float,
+                        default=0.8)
+    a = parser.parse_args()
+
+    global device, hps, h_sr,h_sr48, hps_denoiser
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+    hps = utils.get_hparams_from_file(os.path.join(os.path.split(a.ckpt)[0], 'config.json'))
+    h_sr = utils.get_hparams_from_file(os.path.join(os.path.split(a.ckpt_sr)[0], 'config.json') )
+    h_sr48 = utils.get_hparams_from_file(os.path.join(os.path.split(a.ckpt_sr48)[0], 'config.json') )
+    hps_denoiser = utils.get_hparams_from_file(os.path.join(os.path.split(a.denoiser_ckpt)[0], 'config.json'))
+
+    inference(a)
+
+if __name__ == '__main__':
+    main()

inference_vc.sh

@@ -0,0 +1,11 @@
+# --ckpt "logs/hierspeechpp_libritts460/hierspeechpp_lt460_ckpt.pth" \ LibriTTS-460
+# --ckpt "logs/hierspeechpp_libritts960/hierspeechpp_lt960_ckpt.pth" \ LibriTTS-960
+# --ckpt "logs/hierspeechpp_eng_kor/hierspeechpp_v1_ckpt.pth" \ Large_v1 epoch 60 (paper version)
+# --ckpt "logs/hierspeechpp_eng_kor/hierspeechpp_v2_ckpt.pth" \ Large_v2 epoch 110 (08. Nov. 2023)
+
+CUDA_VISIBLE_DEVICES=0 python3 inference_vc.py \
+                --ckpt "logs/hierspeechpp_eng_kor/hierspeechpp_v2_ckpt.pth" \
+                --output_dir "vc_results_eng_kor_v2" \
+                --noise_scale_vc "0.333" \
+                --noise_scale_ttv "0.333" \
+                --denoise_ratio "0"

logs/hierspeechpp_eng_kor/config.json

@@ -0,0 +1,67 @@
+{
+    "train": {
+      "log_interval": 200,
+      "eval_interval": 10000,
+      "save_interval": 10000,
+      "seed": 1234,
+      "epochs": 20000,
+      "learning_rate": 1e-4,
+      "betas": [0.8, 0.99],
+      "eps": 1e-9,
+      "batch_size": 20,
+      "fp16_run": false,
+      "lr_decay": 0.999,
+      "segment_size": 61440,
+      "init_lr_ratio": 1,
+      "warmup_epochs": 0,
+      "c_mel": 45,
+      "c_kl": 1.0,
+      "c_bi_kl": 0.5,
+      "c_mixup": 1,
+      "c_pho": 45.0,
+      "c_f0": 45,
+      "aug": true
+    },
+    "data": {
+      "train_filelist_path": "filelists/train_eng_kor_wav.txt",
+      "test_filelist_path": "filelists/test_hier_16k_72_wav.txt",
+      "text_cleaners":["english_cleaners2"],
+      "max_wav_value": 32768.0,
+      "sampling_rate": 16000,
+      "filter_length": 1280,
+      "hop_length": 320,
+      "win_length": 1280,
+      "n_mel_channels": 80,
+      "mel_fmin": 0,
+      "mel_fmax": 8000,
+      "add_blank": true,
+      "n_speakers": 0,
+      "cleaned_text": true,
+      "aug_rate": 1.0,
+      "top_db": 20
+    },
+    "model": {
+      "inter_channels": 192,
+      "hidden_channels": 192,
+      "filter_channels": 768,
+      "n_heads": 2,
+      "n_layers": 6,
+      "kernel_size": 3,
+      "p_dropout": 0.1,
+      "resblock": "1",
+      "resblock_kernel_sizes": [3,7,11],
+      "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
+      "upsample_rates": [4,5,4,2,2],
+      "upsample_initial_channel": 512,
+      "upsample_kernel_sizes": [8,11,8,4,4],
+      "n_layers_q": 3,
+      "use_spectral_norm": false,
+      "wav2vec_feature_layer": 7,
+      "gin_channels": 256,
+      "cfg": true,
+      "uncond_ratio": 0.1,
+      "prosody_size": 20
+  
+    }
+  }
+  

logs/hierspeechpp_eng_kor/hierspeechpp_v1_ckpt.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0229501cd273135bce9ee0be09ddef18e079cb8dbe61cda665fec73afbfad0d4
+size 388970075

logs/hierspeechpp_eng_kor/hierspeechpp_v2_ckpt.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54d2aff8f92ff66669ef8a48ee2b604b52190fa2c421e8a20c39596cdebf9c43
+size 388970075

logs/ttv_libritts_v1/config.json

@@ -0,0 +1,55 @@
+{
+    "train": {
+      "log_interval": 200,
+      "eval_interval": 10000,
+      "save_interval": 10000,
+      "seed": 1234,
+      "epochs": 20000,
+      "learning_rate": 2e-4,
+      "betas": [0.8, 0.99],
+      "eps": 1e-9,
+      "batch_size": 32,
+      "fp16_run": false,
+      "lr_decay": 0.999,
+      "segment_size": 192000,
+      "init_lr_ratio": 1,
+      "warmup_epochs": 0,
+      "c_mel": 10,
+      "c_kl": 0.1,
+      "c_pho": 45.0,
+      "c_f0": 1,
+      "aug": true
+    },
+    "data": {
+      "train_filelist_path": "filelists/train_wav.txt",
+      "test_filelist_path": "filelists/test_wav.txt",
+      "text_cleaners":["english_cleaners2"],
+      "max_wav_value": 32768.0,
+      "sampling_rate": 16000,
+      "filter_length": 1280,
+      "hop_length": 320,
+      "win_length": 1280,
+      "n_mel_channels": 80,
+      "mel_fmin": 0,
+      "mel_fmax": 8000,
+      "add_blank": true,
+      "n_speakers": 0,
+      "cleaned_text": true,
+      "aug_rate": 1.0,
+      "top_db": 20
+    },
+    "model": {
+        "inter_channels": 256,
+        "hidden_channels": 256,
+        "filter_channels": 1024,
+        "n_heads": 4,
+        "n_layers": 6,
+        "kernel_size": 3,
+        "p_dropout": 0.1,
+        "resblock": "1",
+        "resblock_kernel_sizes": [3,7,11],
+        "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
+        "use_spectral_norm": false
+      }
+  }
+  

logs/ttv_libritts_v1/ttv_lt960_ckpt.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e408e2c6a72551da0ca616f43e0f989449ff5e0d0ca6229b27dea3dc36aa6c1
+size 434482782

modules.py

@@ -0,0 +1,534 @@
+import math
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+from torch.nn import Conv1d
+from torch.nn.utils import weight_norm, remove_weight_norm
+
+import commons
+from commons import init_weights, get_padding
+from transforms import piecewise_rational_quadratic_transform
+
+from timm.models.vision_transformer import Attention
+from itertools import repeat
+import collections.abc
+
+LRELU_SLOPE = 0.1
+
+class LayerNorm(nn.Module):
+    def __init__(self, channels, eps=1e-5):
+        super().__init__()
+        self.channels = channels
+        self.eps = eps
+
+        self.gamma = nn.Parameter(torch.ones(channels))
+        self.beta = nn.Parameter(torch.zeros(channels))
+
+    def forward(self, x):
+        x = x.transpose(1, -1)
+        x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
+        return x.transpose(1, -1)
+
+
+class ConvReluNorm(nn.Module):
+    def __init__(self, in_channels, hidden_channels, out_channels, kernel_size, n_layers, p_dropout):
+        super().__init__()
+        self.in_channels = in_channels
+        self.hidden_channels = hidden_channels
+        self.out_channels = out_channels
+        self.kernel_size = kernel_size
+        self.n_layers = n_layers
+        self.p_dropout = p_dropout
+        assert n_layers > 1, "Number of layers should be larger than 0."
+
+        self.conv_layers = nn.ModuleList()
+        self.norm_layers = nn.ModuleList()
+        self.conv_layers.append(nn.Conv1d(in_channels, hidden_channels, kernel_size, padding=kernel_size // 2))
+        self.norm_layers.append(LayerNorm(hidden_channels))
+        self.relu_drop = nn.Sequential(
+            nn.ReLU(),
+            nn.Dropout(p_dropout))
+        for _ in range(n_layers - 1):
+            self.conv_layers.append(nn.Conv1d(hidden_channels, hidden_channels, kernel_size, padding=kernel_size // 2))
+            self.norm_layers.append(LayerNorm(hidden_channels))
+        self.proj = nn.Conv1d(hidden_channels, out_channels, 1)
+        self.proj.weight.data.zero_()
+        self.proj.bias.data.zero_()
+
+    def forward(self, x, x_mask):
+        x_org = x
+        for i in range(self.n_layers):
+            x = self.conv_layers[i](x * x_mask)
+            x = self.norm_layers[i](x)
+            x = self.relu_drop(x)
+        x = x_org + self.proj(x)
+        return x * x_mask
+
+
+class DDSConv(nn.Module):
+    """
+    Dialted and Depth-Separable Convolution
+    """
+
+    def __init__(self, channels, kernel_size, n_layers, p_dropout=0.):
+        super().__init__()
+        self.channels = channels
+        self.kernel_size = kernel_size
+        self.n_layers = n_layers
+        self.p_dropout = p_dropout
+
+        self.drop = nn.Dropout(p_dropout)
+        self.convs_sep = nn.ModuleList()
+        self.convs_1x1 = nn.ModuleList()
+        self.norms_1 = nn.ModuleList()
+        self.norms_2 = nn.ModuleList()
+        for i in range(n_layers):
+            dilation = kernel_size ** i
+            padding = (kernel_size * dilation - dilation) // 2
+            self.convs_sep.append(nn.Conv1d(channels, channels, kernel_size,
+                                            groups=channels, dilation=dilation, padding=padding
+                                            ))
+            self.convs_1x1.append(nn.Conv1d(channels, channels, 1))
+            self.norms_1.append(LayerNorm(channels))
+            self.norms_2.append(LayerNorm(channels))
+
+    def forward(self, x, x_mask, g=None):
+        if g is not None:
+            x = x + g
+        for i in range(self.n_layers):
+            y = self.convs_sep[i](x * x_mask)
+            y = self.norms_1[i](y)
+            y = F.gelu(y)
+            y = self.convs_1x1[i](y)
+            y = self.norms_2[i](y)
+            y = F.gelu(y)
+            y = self.drop(y)
+            x = x + y
+        return x * x_mask
+
+
+class WN(torch.nn.Module):
+    def __init__(self, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=0, p_dropout=0):
+        super(WN, self).__init__()
+        assert (kernel_size % 2 == 1)
+        self.hidden_channels = hidden_channels
+        self.kernel_size = kernel_size,
+        self.dilation_rate = dilation_rate
+        self.n_layers = n_layers
+        self.gin_channels = gin_channels
+        self.p_dropout = p_dropout
+
+        self.in_layers = torch.nn.ModuleList()
+        self.res_skip_layers = torch.nn.ModuleList()
+        self.drop = nn.Dropout(p_dropout)
+
+        if gin_channels != 0:
+            cond_layer = torch.nn.Conv1d(gin_channels, 2 * hidden_channels * n_layers, 1)
+            self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight')
+
+        for i in range(n_layers):
+            dilation = dilation_rate ** i
+            padding = int((kernel_size * dilation - dilation) / 2)
+            in_layer = torch.nn.Conv1d(hidden_channels, 2 * hidden_channels, kernel_size,
+                                       dilation=dilation, padding=padding)
+            in_layer = torch.nn.utils.weight_norm(in_layer, name='weight')
+            self.in_layers.append(in_layer)
+
+            # last one is not necessary
+            if i < n_layers - 1:
+                res_skip_channels = 2 * hidden_channels
+            else:
+                res_skip_channels = hidden_channels
+
+            res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1)
+            res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight')
+            self.res_skip_layers.append(res_skip_layer)
+
+    def forward(self, x, x_mask, g=None, **kwargs):
+        output = torch.zeros_like(x)
+        n_channels_tensor = torch.IntTensor([self.hidden_channels])
+
+        if g is not None:
+            g = self.cond_layer(g)
+
+        for i in range(self.n_layers):
+            x_in = self.in_layers[i](x)
+            if g is not None:
+                cond_offset = i * 2 * self.hidden_channels
+                g_l = g[:, cond_offset:cond_offset + 2 * self.hidden_channels, :]
+            else:
+                g_l = torch.zeros_like(x_in)
+
+            acts = commons.fused_add_tanh_sigmoid_multiply(
+                x_in,
+                g_l,
+                n_channels_tensor)
+            acts = self.drop(acts)
+
+            res_skip_acts = self.res_skip_layers[i](acts)
+            if i < self.n_layers - 1:
+                res_acts = res_skip_acts[:, :self.hidden_channels, :]
+                x = (x + res_acts) * x_mask
+                output = output + res_skip_acts[:, self.hidden_channels:, :]
+            else:
+                output = output + res_skip_acts
+        return output * x_mask
+
+    def remove_weight_norm(self):
+        if self.gin_channels != 0:
+            torch.nn.utils.remove_weight_norm(self.cond_layer)
+        for l in self.in_layers:
+            torch.nn.utils.remove_weight_norm(l)
+        for l in self.res_skip_layers:
+            torch.nn.utils.remove_weight_norm(l)
+
+
+class ResBlock1(torch.nn.Module):
+    def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)):
+        super(ResBlock1, self).__init__()
+        self.convs1 = nn.ModuleList([
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
+                               padding=get_padding(kernel_size, dilation[0]))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
+                               padding=get_padding(kernel_size, dilation[1]))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2],
+                               padding=get_padding(kernel_size, dilation[2])))
+        ])
+        self.convs1.apply(init_weights)
+
+        self.convs2 = nn.ModuleList([
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+                               padding=get_padding(kernel_size, 1))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+                               padding=get_padding(kernel_size, 1))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+                               padding=get_padding(kernel_size, 1)))
+        ])
+        self.convs2.apply(init_weights)
+
+    def forward(self, x, x_mask=None):
+        for c1, c2 in zip(self.convs1, self.convs2):
+            xt = F.leaky_relu(x, LRELU_SLOPE)
+            if x_mask is not None:
+                xt = xt * x_mask
+            xt = c1(xt)
+            xt = F.leaky_relu(xt, LRELU_SLOPE)
+            if x_mask is not None:
+                xt = xt * x_mask
+            xt = c2(xt)
+            x = xt + x
+        if x_mask is not None:
+            x = x * x_mask
+        return x
+
+    def remove_weight_norm(self):
+        for l in self.convs1:
+            remove_weight_norm(l)
+        for l in self.convs2:
+            remove_weight_norm(l)
+
+
+class ResBlock2(torch.nn.Module):
+    def __init__(self, channels, kernel_size=3, dilation=(1, 3)):
+        super(ResBlock2, self).__init__()
+        self.convs = nn.ModuleList([
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
+                               padding=get_padding(kernel_size, dilation[0]))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
+                               padding=get_padding(kernel_size, dilation[1])))
+        ])
+        self.convs.apply(init_weights)
+
+    def forward(self, x, x_mask=None):
+        for c in self.convs:
+            xt = F.leaky_relu(x, LRELU_SLOPE)
+            if x_mask is not None:
+                xt = xt * x_mask
+            xt = c(xt)
+            x = xt + x
+        if x_mask is not None:
+            x = x * x_mask
+        return x
+
+    def remove_weight_norm(self):
+        for l in self.convs:
+            remove_weight_norm(l)
+
+
+class Log(nn.Module):
+    def forward(self, x, x_mask, reverse=False, **kwargs):
+        if not reverse:
+            y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask
+            logdet = torch.sum(-y, [1, 2])
+            return y, logdet
+        else:
+            x = torch.exp(x) * x_mask
+            return x
+
+
+class Flip(nn.Module):
+    def forward(self, x, *args, reverse=False, **kwargs):
+        x = torch.flip(x, [1])
+        if not reverse:
+            logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device)
+            return x, logdet
+        else:
+            return x
+
+
+class ElementwiseAffine(nn.Module):
+    def __init__(self, channels):
+        super().__init__()
+        self.channels = channels
+        self.m = nn.Parameter(torch.zeros(channels, 1))
+        self.logs = nn.Parameter(torch.zeros(channels, 1))
+
+    def forward(self, x, x_mask, reverse=False, **kwargs):
+        if not reverse:
+            y = self.m + torch.exp(self.logs) * x
+            y = y * x_mask
+            logdet = torch.sum(self.logs * x_mask, [1, 2])
+            return y, logdet
+        else:
+            x = (x - self.m) * torch.exp(-self.logs) * x_mask
+            return x
+
+
+class ResidualCouplingLayer(nn.Module):
+    def __init__(self,
+                 channels,
+                 hidden_channels,
+                 kernel_size,
+                 dilation_rate,
+                 n_layers,
+                 p_dropout=0,
+                 gin_channels=0,
+                 mean_only=False):
+        assert channels % 2 == 0, "channels should be divisible by 2"
+        super().__init__()
+        self.channels = channels
+        self.hidden_channels = hidden_channels
+        self.kernel_size = kernel_size
+        self.dilation_rate = dilation_rate
+        self.n_layers = n_layers
+        self.half_channels = channels // 2
+        self.mean_only = mean_only
+
+        self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1)
+        self.enc = WN(hidden_channels, kernel_size, dilation_rate, n_layers, p_dropout=p_dropout,
+                      gin_channels=gin_channels)
+        self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1)
+        self.post.weight.data.zero_()
+        self.post.bias.data.zero_()
+
+    def forward(self, x, x_mask, g=None, reverse=False):
+        x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
+        h = self.pre(x0) * x_mask
+        h = self.enc(h, x_mask, g=g)
+        stats = self.post(h) * x_mask
+        if not self.mean_only:
+            m, logs = torch.split(stats, [self.half_channels] * 2, 1)
+        else:
+            m = stats
+            logs = torch.zeros_like(m)
+
+        if not reverse:
+            x1 = m + x1 * torch.exp(logs) * x_mask
+            x = torch.cat([x0, x1], 1)
+            logdet = torch.sum(logs, [1, 2])
+            return x, logdet
+        else:
+            x1 = (x1 - m) * torch.exp(-logs) * x_mask
+            x = torch.cat([x0, x1], 1)
+            return x
+        
+def modulate(x, shift, scale):
+    return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1)
+
+def _ntuple(n):
+    def parse(x):
+        if isinstance(x, collections.abc.Iterable) and not isinstance(x, str):
+            return tuple(x)
+        return tuple(repeat(x, n))
+    return parse
+to_2tuple = _ntuple(2)
+
+class FFN_Conv(nn.Module):
+    """ MLP as used in Vision Transformer, MLP-Mixer and related networks
+    """
+    def __init__(
+            self,
+            in_features,
+            hidden_features=None,
+            out_features=None,
+            act_layer=nn.GELU,
+            norm_layer=None,
+            bias=True,
+            kernel=5,
+            p_dropout=0.1
+    ):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        bias = to_2tuple(bias)
+
+        self.fc1 = nn.Conv1d(in_features, hidden_features, kernel_size=kernel, stride=1, padding=(kernel-1)//2, bias=bias[0])
+        self.act = act_layer()
+        self.norm = norm_layer(hidden_features) if norm_layer is not None else nn.Identity()
+        self.fc2 = nn.Conv1d(hidden_features, out_features, kernel_size=1, bias=bias[1])
+        self.drop = nn.Dropout(p_dropout)
+
+    def forward(self, x, x_mask):
+        x = self.fc1(x.transpose(1,2))
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x*x_mask) * x_mask
+        x = self.drop(x) 
+        return x.transpose(1,2)
+    
+class DiTConVBlock(nn.Module):
+    """
+    A DiT block with adaptive layer norm zero (adaLN-Zero) conditioning.
+    """
+    def __init__(self, hidden_size, num_heads, mlp_ratio=4.0, kernel=9, p_dropout=0.1, **block_kwargs):
+        super().__init__()
+        self.norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
+        self.attn = Attention(hidden_size, num_heads=num_heads, qkv_bias=True, **block_kwargs)
+        self.norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
+        mlp_hidden_dim = int(hidden_size * mlp_ratio)
+        approx_gelu = lambda: nn.GELU(approximate="tanh")
+        self.mlp = FFN_Conv(in_features=hidden_size, hidden_features=mlp_hidden_dim, act_layer=approx_gelu, kernel=kernel, p_dropout=p_dropout)
+        self.adaLN_modulation = nn.Sequential(
+            nn.SiLU(),
+            nn.Linear(hidden_size, 6 * hidden_size, bias=True)
+        )
+    def forward(self, x, c, x_mask):
+        x = x*x_mask
+        shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaLN_modulation(c).chunk(6, dim=1)
+        x = x + gate_msa.unsqueeze(1) * self.attn(modulate(self.norm1(x)*x_mask, shift_msa, scale_msa))*x_mask
+        x = x + gate_mlp.unsqueeze(1) * self.mlp(modulate(self.norm2(x), shift_mlp, scale_mlp), x_mask.transpose(1,2))
+        return x
+            
+class ResidualCouplingLayer_Transformer_simple(nn.Module):
+    def __init__(self,
+                 channels,
+                 hidden_channels,
+                 kernel_size,
+                 dilation_rate,
+                 n_layers,
+                 p_dropout=0.1,
+                 mean_only=False):
+        assert channels % 2 == 0, "channels should be divisible by 2"
+        super().__init__()
+        self.channels = channels
+        self.hidden_channels = hidden_channels
+        self.kernel_size = kernel_size
+        self.dilation_rate = dilation_rate
+        self.n_layers = n_layers
+        self.half_channels = channels // 2
+        self.mean_only = mean_only
+
+        self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1)
+
+        self.enc_block = torch.nn.ModuleList([
+            DiTConVBlock(hidden_channels, 2, mlp_ratio=4.0, kernel=5, p_dropout=p_dropout) for _ in range(n_layers)
+        ])
+
+        self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1)
+        
+        self.initialize_weights()
+
+        self.post.weight.data.zero_()
+        self.post.bias.data.zero_()
+
+    def initialize_weights(self):
+    # Initialize transformer layers:
+        def _basic_init(module):
+            if isinstance(module, (nn.Conv1d, nn.Linear)):
+                torch.nn.init.xavier_uniform_(module.weight)
+                if module.bias is not None:
+                    nn.init.constant_(module.bias, 0)
+
+        self.apply(_basic_init)
+        # Zero-out adaLN modulation layers in DiT blocks:
+        for block in self.enc_block:
+            nn.init.constant_(block.adaLN_modulation[-1].weight, 0)
+            nn.init.constant_(block.adaLN_modulation[-1].bias, 0)
+
+    def forward(self, x, x_mask, g=None, reverse=False):
+        x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
+        h = self.pre(x0) * x_mask
+
+        # h = self.enc(h, x_mask, g=g)
+        h = h.transpose(1,2)
+        x_mask = x_mask.transpose(1,2)
+
+        for blk in self.enc_block:
+            h = blk(h, g, x_mask)
+        
+        x_mask = x_mask.transpose(1,2)
+        h = h.transpose(1,2)
+
+        stats = self.post(h) * x_mask
+        if not self.mean_only:
+            m, logs = torch.split(stats, [self.half_channels] * 2, 1)
+        else:
+            m = stats
+            logs = torch.zeros_like(m)
+
+        if not reverse:
+            x1 = m + x1 * torch.exp(logs) * x_mask
+            x = torch.cat([x0, x1], 1)
+            logdet = torch.sum(logs, [1, 2])
+            return x, logdet
+        else:
+            x1 = (x1 - m) * torch.exp(-logs) * x_mask
+            x = torch.cat([x0, x1], 1)
+            return x
+
+class ConvFlow(nn.Module):
+    def __init__(self, in_channels, filter_channels, kernel_size, n_layers, num_bins=10, tail_bound=5.0):
+        super().__init__()
+        self.in_channels = in_channels
+        self.filter_channels = filter_channels
+        self.kernel_size = kernel_size
+        self.n_layers = n_layers
+        self.num_bins = num_bins
+        self.tail_bound = tail_bound
+        self.half_channels = in_channels // 2
+
+        self.pre = nn.Conv1d(self.half_channels, filter_channels, 1)
+        self.convs = DDSConv(filter_channels, kernel_size, n_layers, p_dropout=0.)
+        self.proj = nn.Conv1d(filter_channels, self.half_channels * (num_bins * 3 - 1), 1)
+        self.proj.weight.data.zero_()
+        self.proj.bias.data.zero_()
+
+    def forward(self, x, x_mask, g=None, reverse=False):
+        x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
+        h = self.pre(x0)
+        h = self.convs(h, x_mask, g=g)
+        h = self.proj(h) * x_mask
+
+        b, c, t = x0.shape
+        h = h.reshape(b, c, -1, t).permute(0, 1, 3, 2)  # [b, cx?, t] -> [b, c, t, ?]
+
+        unnormalized_widths = h[..., :self.num_bins] / math.sqrt(self.filter_channels)
+        unnormalized_heights = h[..., self.num_bins:2 * self.num_bins] / math.sqrt(self.filter_channels)
+        unnormalized_derivatives = h[..., 2 * self.num_bins:]
+
+        x1, logabsdet = piecewise_rational_quadratic_transform(x1,
+                                                               unnormalized_widths,
+                                                               unnormalized_heights,
+                                                               unnormalized_derivatives,
+                                                               inverse=reverse,
+                                                               tails='linear',
+                                                               tail_bound=self.tail_bound
+                                                               )
+
+        x = torch.cat([x0, x1], 1) * x_mask
+        logdet = torch.sum(logabsdet * x_mask, [1, 2])
+        if not reverse:
+            return x, logdet
+        else:
+            return x

requirements.txt

@@ -0,0 +1,15 @@
+AMFM_decompy==1.0.11
+Cython==3.0.3
+einops==0.7.0
+joblib==1.3.2
+matplotlib==3.8.1
+numpy==1.26.1
+pesq==0.0.4
+phonemizer==3.2.1
+scipy==1.11.3
+timm==0.6.13
+torch==1.13.1+cu117
+torchaudio==0.13.1+cu117
+tqdm==4.65.0
+transformers==4.34.0
+Unidecode==1.3.7

speechsr24k/G_340000.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94bc84b6f8bdb375e2027bc7a3222730d29f9d4c042fd9edce0024806d5c4320
+size 1715101

speechsr24k/config.json

@@ -0,0 +1,49 @@
+
+{
+    "train": {
+      "log_interval": 200,
+      "eval_interval": 10000,
+      "save_interval": 10000,
+      "seed": 1234,
+      "epochs": 20000,
+      "learning_rate": 1e-4,
+      "betas": [0.8, 0.99],
+      "eps": 1e-9,
+      "batch_size": 32,
+      "fp16_run": false,
+      "lr_decay": 0.999,
+      "segment_size": 9600,
+      "init_lr_ratio": 1,
+      "warmup_epochs": 0,
+      "c_mel": 45
+    },
+    "data": {
+      "train_filelist_path": "filelists/train_24k_bigvgan_sr.txt",
+      "test_filelist_path": "filelists/test_24k_bigvgan_sr.txt",
+      "text_cleaners":["english_cleaners2"],
+      "max_wav_value": 32768.0,
+      "sampling_rate": 24000,
+      "filter_length": 960,
+      "hop_length": 240,
+      "win_length": 960,
+      "n_mel_channels": 100,
+      "mel_fmin": 0,
+      "mel_fmax": 12000,
+      "add_blank": true,
+      "n_speakers": 0,
+      "cleaned_text": true,
+      "aug_rate": 1.0,
+      "top_db": 20
+    },
+    "model": {
+      "resblock": "0",
+      "resblock_kernel_sizes": [3,7,11],
+      "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
+      "upsample_rates": [3],
+      "upsample_initial_channel": 32,
+      "upsample_kernel_sizes": [3],
+      "use_spectral_norm": false
+
+    }
+  }
+  

speechsr24k/speechsr.py

@@ -0,0 +1,253 @@
+import torch
+from torch import nn
+from torch.nn import functional as F
+import modules
+
+from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d
+from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
+from commons import init_weights, get_padding
+from torch.cuda.amp import autocast
+import torchaudio
+from einops import rearrange
+
+from alias_free_torch import *
+import activations
+
+class AMPBlock0(torch.nn.Module):
+    def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5), activation=None):
+        super(AMPBlock0, self).__init__()
+      
+        self.convs1 = nn.ModuleList([
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
+                               padding=get_padding(kernel_size, dilation[0]))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
+                               padding=get_padding(kernel_size, dilation[1]))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2],
+                               padding=get_padding(kernel_size, dilation[2]))),
+        ])
+        self.convs1.apply(init_weights)
+
+        self.convs2 = nn.ModuleList([
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+                               padding=get_padding(kernel_size, 1))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+                               padding=get_padding(kernel_size, 1))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+                               padding=get_padding(kernel_size, 1))),
+        ])
+        self.convs2.apply(init_weights)
+
+        self.num_layers = len(self.convs1) + len(self.convs2) # total number of conv layers
+
+
+        self.activations = nn.ModuleList([
+            Activation1d(
+                activation=activations.SnakeBeta(channels, alpha_logscale=True))
+                for _ in range(self.num_layers)
+        ])
+  
+    def forward(self, x):
+        acts1, acts2 = self.activations[::2], self.activations[1::2]
+        for c1, c2, a1, a2 in zip(self.convs1, self.convs2, acts1, acts2):
+            xt = a1(x)
+            xt = c1(xt)
+            xt = a2(xt)
+            xt = c2(xt)
+            x = xt + x
+
+        return x
+
+    def remove_weight_norm(self):
+        for l in self.convs1:
+            remove_weight_norm(l)
+        for l in self.convs2:
+            remove_weight_norm(l)
+
+
+class Generator(torch.nn.Module):
+    def __init__(self, initial_channel, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gin_channels=0):
+        super(Generator, self).__init__()
+        self.num_kernels = len(resblock_kernel_sizes)
+        self.num_upsamples = len(upsample_rates)
+        
+        self.conv_pre = weight_norm(Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3))
+        resblock = AMPBlock0
+
+        self.resblocks = nn.ModuleList()
+        for i in range(1):
+            ch = upsample_initial_channel//(2**(i))
+            for j, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)):
+                self.resblocks.append(resblock(ch, k, d, activation="snakebeta"))
+
+        activation_post = activations.SnakeBeta(ch, alpha_logscale=True)
+        self.activation_post = Activation1d(activation=activation_post)
+
+        self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
+        if gin_channels != 0:
+            self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)
+
+    def forward(self, x, g=None):
+        x = self.conv_pre(x)
+        if g is not None:
+          x = x + self.cond(g)
+
+        for i in range(self.num_upsamples):
+      
+            x = F.interpolate(x, int(x.shape[-1] * 1.5), mode='linear')
+            xs = None
+            for j in range(self.num_kernels):
+                if xs is None:
+                    xs = self.resblocks[i*self.num_kernels+j](x)
+                else:
+                    xs += self.resblocks[i*self.num_kernels+j](x)
+            x = xs / self.num_kernels
+
+        x = self.activation_post(x)
+        x = self.conv_post(x)
+        x = torch.tanh(x)
+
+        return x
+
+    def remove_weight_norm(self):
+        print('Removing weight norm...')
+        for l in self.resblocks:
+            l.remove_weight_norm()
+        remove_weight_norm(self.conv_pre)
+
+class DiscriminatorP(torch.nn.Module):
+    def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
+        super(DiscriminatorP, self).__init__()
+        self.period = period
+        self.use_spectral_norm = use_spectral_norm
+        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
+        self.convs = nn.ModuleList([
+            norm_f(Conv2d(1, 32, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(Conv2d(32, 128, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(Conv2d(128, 512, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(Conv2d(512, 1024, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(Conv2d(1024, 1024, (kernel_size, 1), 1, padding=(get_padding(kernel_size, 1), 0))),
+        ])
+        self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
+
+    def forward(self, x):
+        fmap = []
+
+        # 1d to 2d
+        b, c, t = x.shape
+        if t % self.period != 0: # pad first
+            n_pad = self.period - (t % self.period)
+            x = F.pad(x, (0, n_pad), "reflect")
+            t = t + n_pad
+        x = x.view(b, c, t // self.period, self.period)
+
+        for l in self.convs:
+            x = l(x)
+            x = F.leaky_relu(x, modules.LRELU_SLOPE)
+            fmap.append(x)
+        x = self.conv_post(x)
+        fmap.append(x)
+        x = torch.flatten(x, 1, -1)
+
+        return x, fmap
+
+class DiscriminatorR(torch.nn.Module):
+    def __init__(self, resolution, use_spectral_norm=False):
+        super(DiscriminatorR, self).__init__()
+        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
+
+        n_fft, hop_length, win_length = resolution
+        self.spec_transform = torchaudio.transforms.Spectrogram(
+            n_fft=n_fft, hop_length=hop_length, win_length=win_length, window_fn=torch.hann_window,
+            normalized=True, center=False, pad_mode=None, power=None)
+
+        self.convs = nn.ModuleList([
+            norm_f(nn.Conv2d(2, 32, (3, 9), padding=(1, 4))),
+            norm_f(nn.Conv2d(32, 32, (3, 9), stride=(1, 2), padding=(1, 4))),
+            norm_f(nn.Conv2d(32, 32, (3, 9), stride=(1, 2), dilation=(2,1), padding=(2, 4))),
+            norm_f(nn.Conv2d(32, 32, (3, 9), stride=(1, 2), dilation=(4,1), padding=(4, 4))),
+            norm_f(nn.Conv2d(32, 32, (3, 3), padding=(1, 1))),
+        ])
+        self.conv_post = norm_f(nn.Conv2d(32, 1, (3, 3), padding=(1, 1)))
+
+    def forward(self, y):
+        fmap = []
+
+        x = self.spec_transform(y)  # [B, 2, Freq, Frames, 2]
+        x = torch.cat([x.real, x.imag], dim=1)
+        x = rearrange(x, 'b c w t -> b c t w')
+
+        for l in self.convs:
+            x = l(x)
+            x = F.leaky_relu(x, modules.LRELU_SLOPE)
+            fmap.append(x)
+        x = self.conv_post(x)
+        fmap.append(x)
+        x = torch.flatten(x, 1, -1)
+
+        return x, fmap
+
+
+class MultiPeriodDiscriminator(torch.nn.Module):
+    def __init__(self, use_spectral_norm=False):
+        super(MultiPeriodDiscriminator, self).__init__()
+        periods = [2,3,5,7,11]
+        resolutions = [[2048, 512, 2048], [1024, 256, 1024], [512, 128, 512], [256, 64, 256], [128, 32, 128]]
+
+        discs = [DiscriminatorR(resolutions[i], use_spectral_norm=use_spectral_norm) for i in range(len(resolutions))]
+        discs = discs + [DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods]
+        self.discriminators = nn.ModuleList(discs)
+
+    def forward(self, y, y_hat):
+        y_d_rs = []
+        y_d_gs = []
+        fmap_rs = []
+        fmap_gs = []
+        for i, d in enumerate(self.discriminators):
+            y_d_r, fmap_r = d(y)
+            y_d_g, fmap_g = d(y_hat)
+            y_d_rs.append(y_d_r)
+            y_d_gs.append(y_d_g)
+            fmap_rs.append(fmap_r)
+            fmap_gs.append(fmap_g)
+
+        return y_d_rs, y_d_gs, fmap_rs, fmap_gs
+
+class SynthesizerTrn(nn.Module):
+  """
+  Synthesizer for Training
+  """
+
+  def __init__(self,
+
+    spec_channels,
+    segment_size,
+    resblock,
+    resblock_kernel_sizes,
+    resblock_dilation_sizes,
+    upsample_rates,
+    upsample_initial_channel,
+    upsample_kernel_sizes,
+    **kwargs):
+
+    super().__init__()
+    self.spec_channels = spec_channels
+    self.resblock = resblock
+    self.resblock_kernel_sizes = resblock_kernel_sizes
+    self.resblock_dilation_sizes = resblock_dilation_sizes
+    self.upsample_rates = upsample_rates
+    self.upsample_initial_channel = upsample_initial_channel
+    self.upsample_kernel_sizes = upsample_kernel_sizes
+    self.segment_size = segment_size
+
+    self.dec = Generator(1, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes)
+
+  def forward(self, x):
+
+    y = self.dec(x)
+    return y
+  @torch.no_grad()
+  def infer(self, x, max_len=None):
+
+    o = self.dec(x[:,:,:max_len])
+    return o
+

speechsr48k/G_100000.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62c70874ac4efeb4dc9c8aa9dc0a611a951e1c36292abeb4c406d7fb91e0eefc
+size 1715101

speechsr48k/config.json

@@ -0,0 +1,48 @@
+
+{
+    "train": {
+      "log_interval": 200,
+      "eval_interval": 10000,
+      "save_interval": 10000,
+      "seed": 1234,
+      "epochs": 20000,
+      "learning_rate": 1e-4,
+      "betas": [0.8, 0.99],
+      "eps": 1e-9,
+      "batch_size": 32,
+      "fp16_run": false,
+      "lr_decay": 0.995,
+      "segment_size": 9600,
+      "init_lr_ratio": 1,
+      "warmup_epochs": 0,
+      "c_mel": 45
+    },
+    "data": {
+      "train_filelist_path": "filelists/train_48k_vctk_trim_bigvgan_sr.txt",
+      "test_filelist_path": "filelists/test_48k_vctk_trim_bigvgan_sr.txt",
+      "text_cleaners":["english_cleaners2"],
+      "max_wav_value": 32768.0,
+      "sampling_rate": 48000,
+      "filter_length": 1280,
+      "hop_length": 320,
+      "win_length": 1280,
+      "n_mel_channels": 128,
+      "mel_fmin": 0,
+      "mel_fmax": 24000,
+      "add_blank": true,
+      "n_speakers": 0,
+      "cleaned_text": true,
+      "aug_rate": 1.0,
+      "top_db": 20
+    },
+    "model": {
+      "resblock": "0",
+      "resblock_kernel_sizes": [3,7,11],
+      "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
+      "upsample_rates": [3],
+      "upsample_initial_channel": 32,
+      "upsample_kernel_sizes": [3],
+      "use_spectral_norm": false
+    }
+  }
+  

speechsr48k/speechsr.py

@@ -0,0 +1,252 @@
+import torch
+from torch import nn
+from torch.nn import functional as F
+import modules
+
+from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d
+from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
+from commons import init_weights, get_padding
+from torch.cuda.amp import autocast
+import torchaudio
+from einops import rearrange
+
+from alias_free_torch import *
+import activations
+
+class AMPBlock0(torch.nn.Module):
+    def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5), activation=None):
+        super(AMPBlock0, self).__init__()
+      
+        self.convs1 = nn.ModuleList([
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
+                               padding=get_padding(kernel_size, dilation[0]))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
+                               padding=get_padding(kernel_size, dilation[1]))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2],
+                               padding=get_padding(kernel_size, dilation[2]))),
+        ])
+        self.convs1.apply(init_weights)
+
+        self.convs2 = nn.ModuleList([
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+                               padding=get_padding(kernel_size, 1))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+                               padding=get_padding(kernel_size, 1))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+                               padding=get_padding(kernel_size, 1))),
+        ])
+        self.convs2.apply(init_weights)
+
+        self.num_layers = len(self.convs1) + len(self.convs2) # total number of conv layers
+
+
+        self.activations = nn.ModuleList([
+            Activation1d(
+                activation=activations.SnakeBeta(channels, alpha_logscale=True))
+                for _ in range(self.num_layers)
+        ])
+  
+    def forward(self, x):
+        acts1, acts2 = self.activations[::2], self.activations[1::2]
+        for c1, c2, a1, a2 in zip(self.convs1, self.convs2, acts1, acts2):
+            xt = a1(x)
+            xt = c1(xt)
+            xt = a2(xt)
+            xt = c2(xt)
+            x = xt + x
+
+        return x
+
+    def remove_weight_norm(self):
+        for l in self.convs1:
+            remove_weight_norm(l)
+        for l in self.convs2:
+            remove_weight_norm(l)
+
+
+class Generator(torch.nn.Module):
+    def __init__(self, initial_channel, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gin_channels=0):
+        super(Generator, self).__init__()
+        self.num_kernels = len(resblock_kernel_sizes)
+        self.num_upsamples = len(upsample_rates)
+        
+        self.conv_pre = weight_norm(Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3))
+        resblock = AMPBlock0
+
+        self.resblocks = nn.ModuleList()
+        for i in range(1):
+            ch = upsample_initial_channel//(2**(i))
+            for j, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)):
+                self.resblocks.append(resblock(ch, k, d, activation="snakebeta"))
+
+        activation_post = activations.SnakeBeta(ch, alpha_logscale=True)
+        self.activation_post = Activation1d(activation=activation_post)
+
+        self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
+        if gin_channels != 0:
+            self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)
+
+    def forward(self, x, g=None):
+        x = self.conv_pre(x)
+        if g is not None:
+          x = x + self.cond(g)
+
+        for i in range(self.num_upsamples):
+      
+            x = F.interpolate(x, int(x.shape[-1] * 3), mode='linear')
+            xs = None
+            for j in range(self.num_kernels):
+                if xs is None:
+                    xs = self.resblocks[i*self.num_kernels+j](x)
+                else:
+                    xs += self.resblocks[i*self.num_kernels+j](x)
+            x = xs / self.num_kernels
+
+        x = self.activation_post(x)
+        x = self.conv_post(x)
+        x = torch.tanh(x)
+
+        return x
+
+    def remove_weight_norm(self):
+        print('Removing weight norm...')
+        for l in self.resblocks:
+            l.remove_weight_norm()
+
+class DiscriminatorP(torch.nn.Module):
+    def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
+        super(DiscriminatorP, self).__init__()
+        self.period = period
+        self.use_spectral_norm = use_spectral_norm
+        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
+        self.convs = nn.ModuleList([
+            norm_f(Conv2d(1, 32, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(Conv2d(32, 128, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(Conv2d(128, 512, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(Conv2d(512, 1024, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(Conv2d(1024, 1024, (kernel_size, 1), 1, padding=(get_padding(kernel_size, 1), 0))),
+        ])
+        self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
+
+    def forward(self, x):
+        fmap = []
+
+        # 1d to 2d
+        b, c, t = x.shape
+        if t % self.period != 0: # pad first
+            n_pad = self.period - (t % self.period)
+            x = F.pad(x, (0, n_pad), "reflect")
+            t = t + n_pad
+        x = x.view(b, c, t // self.period, self.period)
+
+        for l in self.convs:
+            x = l(x)
+            x = F.leaky_relu(x, modules.LRELU_SLOPE)
+            fmap.append(x)
+        x = self.conv_post(x)
+        fmap.append(x)
+        x = torch.flatten(x, 1, -1)
+
+        return x, fmap
+
+class DiscriminatorR(torch.nn.Module):
+    def __init__(self, resolution, use_spectral_norm=False):
+        super(DiscriminatorR, self).__init__()
+        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
+
+        n_fft, hop_length, win_length = resolution
+        self.spec_transform = torchaudio.transforms.Spectrogram(
+            n_fft=n_fft, hop_length=hop_length, win_length=win_length, window_fn=torch.hann_window,
+            normalized=True, center=False, pad_mode=None, power=None)
+
+        self.convs = nn.ModuleList([
+            norm_f(nn.Conv2d(2, 32, (3, 9), padding=(1, 4))),
+            norm_f(nn.Conv2d(32, 32, (3, 9), stride=(1, 2), padding=(1, 4))),
+            norm_f(nn.Conv2d(32, 32, (3, 9), stride=(1, 2), dilation=(2,1), padding=(2, 4))),
+            norm_f(nn.Conv2d(32, 32, (3, 9), stride=(1, 2), dilation=(4,1), padding=(4, 4))),
+            norm_f(nn.Conv2d(32, 32, (3, 3), padding=(1, 1))),
+        ])
+        self.conv_post = norm_f(nn.Conv2d(32, 1, (3, 3), padding=(1, 1)))
+
+    def forward(self, y):
+        fmap = []
+
+        x = self.spec_transform(y)  # [B, 2, Freq, Frames, 2]
+        x = torch.cat([x.real, x.imag], dim=1)
+        x = rearrange(x, 'b c w t -> b c t w')
+
+        for l in self.convs:
+            x = l(x)
+            x = F.leaky_relu(x, modules.LRELU_SLOPE)
+            fmap.append(x)
+        x = self.conv_post(x)
+        fmap.append(x)
+        x = torch.flatten(x, 1, -1)
+
+        return x, fmap
+
+
+class MultiPeriodDiscriminator(torch.nn.Module):
+    def __init__(self, use_spectral_norm=False):
+        super(MultiPeriodDiscriminator, self).__init__()
+        periods = [2,3,5,7,11]
+        resolutions = [[4096, 1024, 4096], [2048, 512, 2048], [1024, 256, 1024], [512, 128, 512], [256, 64, 256], [128, 32, 128]]
+
+        discs = [DiscriminatorR(resolutions[i], use_spectral_norm=use_spectral_norm) for i in range(len(resolutions))]
+        discs = discs + [DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods]
+        self.discriminators = nn.ModuleList(discs)
+
+    def forward(self, y, y_hat):
+        y_d_rs = []
+        y_d_gs = []
+        fmap_rs = []
+        fmap_gs = []
+        for i, d in enumerate(self.discriminators):
+            y_d_r, fmap_r = d(y)
+            y_d_g, fmap_g = d(y_hat)
+            y_d_rs.append(y_d_r)
+            y_d_gs.append(y_d_g)
+            fmap_rs.append(fmap_r)
+            fmap_gs.append(fmap_g)
+
+        return y_d_rs, y_d_gs, fmap_rs, fmap_gs
+
+class SynthesizerTrn(nn.Module):
+  """
+  Synthesizer for Training
+  """
+
+  def __init__(self,
+
+    spec_channels,
+    segment_size,
+    resblock,
+    resblock_kernel_sizes,
+    resblock_dilation_sizes,
+    upsample_rates,
+    upsample_initial_channel,
+    upsample_kernel_sizes,
+    **kwargs):
+
+    super().__init__()
+    self.spec_channels = spec_channels
+    self.resblock = resblock
+    self.resblock_kernel_sizes = resblock_kernel_sizes
+    self.resblock_dilation_sizes = resblock_dilation_sizes
+    self.upsample_rates = upsample_rates
+    self.upsample_initial_channel = upsample_initial_channel
+    self.upsample_kernel_sizes = upsample_kernel_sizes
+    self.segment_size = segment_size
+
+    self.dec = Generator(1, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes)
+
+  def forward(self, x):
+
+    y = self.dec(x)
+    return y
+  @torch.no_grad()
+  def infer(self, x, max_len=None):
+
+    o = self.dec(x[:,:,:max_len])
+    return o
+

styleencoder.py

@@ -0,0 +1,91 @@
+import attentions
+from torch import nn
+import torch
+from torch.nn import functional as F
+
+class Mish(nn.Module):
+    def __init__(self):
+        super(Mish, self).__init__()
+    def forward(self, x):
+        return x * torch.tanh(F.softplus(x))
+
+
+class Conv1dGLU(nn.Module):
+    '''
+    Conv1d + GLU(Gated Linear Unit) with residual connection.
+    For GLU refer to https://arxiv.org/abs/1612.08083 paper.
+    '''
+
+    def __init__(self, in_channels, out_channels, kernel_size, dropout):
+        super(Conv1dGLU, self).__init__()
+        self.out_channels = out_channels
+        self.conv1 = nn.Conv1d(in_channels, 2 * out_channels, kernel_size=kernel_size, padding=2)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x):
+        residual = x
+        x = self.conv1(x)
+        x1, x2 = torch.split(x, split_size_or_sections=self.out_channels, dim=1)
+        x = x1 * torch.sigmoid(x2)
+        x = residual + self.dropout(x)
+        return x
+
+class StyleEncoder(torch.nn.Module):
+    def __init__(self, in_dim=513, hidden_dim=128, out_dim=256):
+
+        super().__init__()
+
+        self.in_dim = in_dim # Linear 513 wav2vec 2.0 1024
+        self.hidden_dim = hidden_dim
+        self.out_dim = out_dim
+        self.kernel_size = 5
+        self.n_head = 2
+        self.dropout = 0.1
+
+        self.spectral = nn.Sequential(
+            nn.Conv1d(self.in_dim, self.hidden_dim, 1),
+            Mish(),
+            nn.Dropout(self.dropout),
+            nn.Conv1d(self.hidden_dim, self.hidden_dim, 1),
+            Mish(),
+            nn.Dropout(self.dropout)
+        )
+
+        self.temporal = nn.Sequential(
+            Conv1dGLU(self.hidden_dim, self.hidden_dim, self.kernel_size, self.dropout),
+            Conv1dGLU(self.hidden_dim, self.hidden_dim, self.kernel_size, self.dropout),
+        )
+
+        self.slf_attn = attentions.MultiHeadAttention(self.hidden_dim, self.hidden_dim, self.n_head, p_dropout = self.dropout, proximal_bias= False, proximal_init=True)
+        self.atten_drop = nn.Dropout(self.dropout)
+        self.fc = nn.Conv1d(self.hidden_dim, self.out_dim, 1)
+
+    def forward(self, x, mask=None):
+
+        # spectral
+        x = self.spectral(x)*mask
+        # temporal
+        x = self.temporal(x)*mask
+
+        # self-attention
+        attn_mask = mask.unsqueeze(2) * mask.unsqueeze(-1)
+        y = self.slf_attn(x,x, attn_mask=attn_mask)
+        x = x+ self.atten_drop(y)
+
+        # fc
+        x = self.fc(x)
+
+        # temoral average pooling
+        w = self.temporal_avg_pool(x, mask=mask)
+
+        return w
+
+    def temporal_avg_pool(self, x, mask=None):
+        if mask is None:
+            out = torch.mean(x, dim=2)
+        else:
+            len_ = mask.sum(dim=2)
+            x = x.sum(dim=2)
+
+            out = torch.div(x, len_)
+        return out