Spaces:
Running
Running
| # -*- coding: utf-8 -*- | |
| # Copyright 2019 Tomoki Hayashi | |
| # MIT License (https://opensource.org/licenses/MIT) | |
| import torch | |
| import torch.nn.functional as F | |
| from torch import nn | |
| from ..hparams import HParams | |
| def _make_stft_cfg(hop_length, win_length=None): | |
| if win_length is None: | |
| win_length = 4 * hop_length | |
| n_fft = 2 ** (win_length - 1).bit_length() | |
| return dict(n_fft=n_fft, hop_length=hop_length, win_length=win_length) | |
| def get_stft_cfgs(hp: HParams): | |
| assert hp.wav_rate == 44100, f"wav_rate must be 44100, got {hp.wav_rate}" | |
| return [_make_stft_cfg(h) for h in (100, 256, 512)] | |
| def stft(x, n_fft, hop_length, win_length, window): | |
| dtype = x.dtype | |
| x = torch.stft(x.float(), n_fft, hop_length, win_length, window, return_complex=True) | |
| x = x.abs().to(dtype) | |
| x = x.transpose(2, 1) # (b f t) -> (b t f) | |
| return x | |
| class SpectralConvergengeLoss(nn.Module): | |
| def forward(self, x_mag, y_mag): | |
| """Calculate forward propagation. | |
| Args: | |
| x_mag (Tensor): Magnitude spectrogram of predicted signal (B, #frames, #freq_bins). | |
| y_mag (Tensor): Magnitude spectrogram of groundtruth signal (B, #frames, #freq_bins). | |
| Returns: | |
| Tensor: Spectral convergence loss value. | |
| """ | |
| return torch.norm(y_mag - x_mag, p="fro") / torch.norm(y_mag, p="fro") | |
| class LogSTFTMagnitudeLoss(nn.Module): | |
| def forward(self, x_mag, y_mag): | |
| """Calculate forward propagation. | |
| Args: | |
| x_mag (Tensor): Magnitude spectrogram of predicted signal (B, #frames, #freq_bins). | |
| y_mag (Tensor): Magnitude spectrogram of groundtruth signal (B, #frames, #freq_bins). | |
| Returns: | |
| Tensor: Log STFT magnitude loss value. | |
| """ | |
| return F.l1_loss(torch.log1p(x_mag), torch.log1p(y_mag)) | |
| class STFTLoss(nn.Module): | |
| def __init__(self, hp, stft_cfg: dict, window="hann_window"): | |
| super().__init__() | |
| self.hp = hp | |
| self.stft_cfg = stft_cfg | |
| self.spectral_convergenge_loss = SpectralConvergengeLoss() | |
| self.log_stft_magnitude_loss = LogSTFTMagnitudeLoss() | |
| self.register_buffer("window", getattr(torch, window)(stft_cfg["win_length"]), persistent=False) | |
| def forward(self, x, y): | |
| """Calculate forward propagation. | |
| Args: | |
| x (Tensor): Predicted signal (B, T). | |
| y (Tensor): Groundtruth signal (B, T). | |
| Returns: | |
| Tensor: Spectral convergence loss value. | |
| Tensor: Log STFT magnitude loss value. | |
| """ | |
| stft_cfg = dict(self.stft_cfg) | |
| x_mag = stft(x, **stft_cfg, window=self.window) # (b t) -> (b t f) | |
| y_mag = stft(y, **stft_cfg, window=self.window) | |
| sc_loss = self.spectral_convergenge_loss(x_mag, y_mag) | |
| mag_loss = self.log_stft_magnitude_loss(x_mag, y_mag) | |
| return dict(sc=sc_loss, mag=mag_loss) | |
| class MRSTFTLoss(nn.Module): | |
| def __init__(self, hp: HParams, window="hann_window"): | |
| """Initialize Multi resolution STFT loss module. | |
| Args: | |
| resolutions (list): List of (FFT size, hop size, window length). | |
| window (str): Window function type. | |
| """ | |
| super().__init__() | |
| stft_cfgs = get_stft_cfgs(hp) | |
| self.stft_losses = nn.ModuleList() | |
| self.hp = hp | |
| for c in stft_cfgs: | |
| self.stft_losses += [STFTLoss(hp, c, window=window)] | |
| def forward(self, x, y): | |
| """Calculate forward propagation. | |
| Args: | |
| x (Tensor): Predicted signal (b t). | |
| y (Tensor): Groundtruth signal (b t). | |
| Returns: | |
| Tensor: Multi resolution spectral convergence loss value. | |
| Tensor: Multi resolution log STFT magnitude loss value. | |
| """ | |
| assert x.dim() == 2 and y.dim() == 2, f"(b t) is expected, but got {x.shape} and {y.shape}." | |
| dtype = x.dtype | |
| x = x.float() | |
| y = y.float() | |
| # Align length | |
| x = x[..., : y.shape[-1]] | |
| y = y[..., : x.shape[-1]] | |
| losses = {} | |
| for f in self.stft_losses: | |
| d = f(x, y) | |
| for k, v in d.items(): | |
| losses.setdefault(k, []).append(v) | |
| for k, v in losses.items(): | |
| losses[k] = torch.stack(v, dim=0).mean().to(dtype) | |
| return losses | |