changes

src/tts_vits/Readme.md

@@ -1,22 +0,0 @@
-## 凉宫春日的vits变声
-
-### 安装环境
-
-```
-pip install -r requirements.txt
-```
-
-## 模型下载
-
-[模型](https://huggingface.co/scixing/Haruhi_Vits/blob/main/Haruhi_54000.pth)该模型在程序中使用set_model_path加载
-
-[hubert模型](https://huggingface.co/scixing/Haruhi_Vits/blob/main/hubert-soft-0d54a1f4.pt)该模型放入`tts_vits\hubert`文件夹下
-
-## 使用方法
-
-```python
-# 设置模型路径
-set_model_path("vits_models/Haruhi_54000.pth")
-# 生成语音 第一个参数为文本 第二个参数为音高
-vits_haruhi("真実はいつもひとつ", 8)
-```

src/tts_vits/attentions.py

@@ -1,303 +0,0 @@
-import copy
-import math
-import numpy as np
-import torch
-from torch import nn
-from torch.nn import functional as F
-
-import commons
-import modules
-from modules import LayerNorm
-   
-
-class Encoder(nn.Module):
-  def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., window_size=4, **kwargs):
-    super().__init__()
-    self.hidden_channels = hidden_channels
-    self.filter_channels = filter_channels
-    self.n_heads = n_heads
-    self.n_layers = n_layers
-    self.kernel_size = kernel_size
-    self.p_dropout = p_dropout
-    self.window_size = window_size
-
-    self.drop = nn.Dropout(p_dropout)
-    self.attn_layers = nn.ModuleList()
-    self.norm_layers_1 = nn.ModuleList()
-    self.ffn_layers = nn.ModuleList()
-    self.norm_layers_2 = nn.ModuleList()
-    for i in range(self.n_layers):
-      self.attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, window_size=window_size))
-      self.norm_layers_1.append(LayerNorm(hidden_channels))
-      self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout))
-      self.norm_layers_2.append(LayerNorm(hidden_channels))
-
-  def forward(self, x, x_mask):
-    attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
-    x = x * x_mask
-    for i in range(self.n_layers):
-      y = self.attn_layers[i](x, x, attn_mask)
-      y = self.drop(y)
-      x = self.norm_layers_1[i](x + y)
-
-      y = self.ffn_layers[i](x, x_mask)
-      y = self.drop(y)
-      x = self.norm_layers_2[i](x + y)
-    x = x * x_mask
-    return x
-
-
-class Decoder(nn.Module):
-  def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., proximal_bias=False, proximal_init=True, **kwargs):
-    super().__init__()
-    self.hidden_channels = hidden_channels
-    self.filter_channels = filter_channels
-    self.n_heads = n_heads
-    self.n_layers = n_layers
-    self.kernel_size = kernel_size
-    self.p_dropout = p_dropout
-    self.proximal_bias = proximal_bias
-    self.proximal_init = proximal_init
-
-    self.drop = nn.Dropout(p_dropout)
-    self.self_attn_layers = nn.ModuleList()
-    self.norm_layers_0 = nn.ModuleList()
-    self.encdec_attn_layers = nn.ModuleList()
-    self.norm_layers_1 = nn.ModuleList()
-    self.ffn_layers = nn.ModuleList()
-    self.norm_layers_2 = nn.ModuleList()
-    for i in range(self.n_layers):
-      self.self_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, proximal_bias=proximal_bias, proximal_init=proximal_init))
-      self.norm_layers_0.append(LayerNorm(hidden_channels))
-      self.encdec_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout))
-      self.norm_layers_1.append(LayerNorm(hidden_channels))
-      self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout, causal=True))
-      self.norm_layers_2.append(LayerNorm(hidden_channels))
-
-  def forward(self, x, x_mask, h, h_mask):
-    """
-    x: decoder input
-    h: encoder output
-    """
-    self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(device=x.device, dtype=x.dtype)
-    encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
-    x = x * x_mask
-    for i in range(self.n_layers):
-      y = self.self_attn_layers[i](x, x, self_attn_mask)
-      y = self.drop(y)
-      x = self.norm_layers_0[i](x + y)
-
-      y = self.encdec_attn_layers[i](x, h, encdec_attn_mask)
-      y = self.drop(y)
-      x = self.norm_layers_1[i](x + y)
-      
-      y = self.ffn_layers[i](x, x_mask)
-      y = self.drop(y)
-      x = self.norm_layers_2[i](x + y)
-    x = x * x_mask
-    return x
-
-
-class MultiHeadAttention(nn.Module):
-  def __init__(self, channels, out_channels, n_heads, p_dropout=0., window_size=None, heads_share=True, block_length=None, proximal_bias=False, proximal_init=False):
-    super().__init__()
-    assert channels % n_heads == 0
-
-    self.channels = channels
-    self.out_channels = out_channels
-    self.n_heads = n_heads
-    self.p_dropout = p_dropout
-    self.window_size = window_size
-    self.heads_share = heads_share
-    self.block_length = block_length
-    self.proximal_bias = proximal_bias
-    self.proximal_init = proximal_init
-    self.attn = None
-
-    self.k_channels = channels // n_heads
-    self.conv_q = nn.Conv1d(channels, channels, 1)
-    self.conv_k = nn.Conv1d(channels, channels, 1)
-    self.conv_v = nn.Conv1d(channels, channels, 1)
-    self.conv_o = nn.Conv1d(channels, out_channels, 1)
-    self.drop = nn.Dropout(p_dropout)
-
-    if window_size is not None:
-      n_heads_rel = 1 if heads_share else n_heads
-      rel_stddev = self.k_channels**-0.5
-      self.emb_rel_k = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev)
-      self.emb_rel_v = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev)
-
-    nn.init.xavier_uniform_(self.conv_q.weight)
-    nn.init.xavier_uniform_(self.conv_k.weight)
-    nn.init.xavier_uniform_(self.conv_v.weight)
-    if proximal_init:
-      with torch.no_grad():
-        self.conv_k.weight.copy_(self.conv_q.weight)
-        self.conv_k.bias.copy_(self.conv_q.bias)
-      
-  def forward(self, x, c, attn_mask=None):
-    q = self.conv_q(x)
-    k = self.conv_k(c)
-    v = self.conv_v(c)
-    
-    x, self.attn = self.attention(q, k, v, mask=attn_mask)
-
-    x = self.conv_o(x)
-    return x
-
-  def attention(self, query, key, value, mask=None):
-    # reshape [b, d, t] -> [b, n_h, t, d_k]
-    b, d, t_s, t_t = (*key.size(), query.size(2))
-    query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3)
-    key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
-    value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
-
-    scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1))
-    if self.window_size is not None:
-      assert t_s == t_t, "Relative attention is only available for self-attention."
-      key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s)
-      rel_logits = self._matmul_with_relative_keys(query /math.sqrt(self.k_channels), key_relative_embeddings)
-      scores_local = self._relative_position_to_absolute_position(rel_logits)
-      scores = scores + scores_local
-    if self.proximal_bias:
-      assert t_s == t_t, "Proximal bias is only available for self-attention."
-      scores = scores + self._attention_bias_proximal(t_s).to(device=scores.device, dtype=scores.dtype)
-    if mask is not None:
-      scores = scores.masked_fill(mask == 0, -1e4)
-      if self.block_length is not None:
-        assert t_s == t_t, "Local attention is only available for self-attention."
-        block_mask = torch.ones_like(scores).triu(-self.block_length).tril(self.block_length)
-        scores = scores.masked_fill(block_mask == 0, -1e4)
-    p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s]
-    p_attn = self.drop(p_attn)
-    output = torch.matmul(p_attn, value)
-    if self.window_size is not None:
-      relative_weights = self._absolute_position_to_relative_position(p_attn)
-      value_relative_embeddings = self._get_relative_embeddings(self.emb_rel_v, t_s)
-      output = output + self._matmul_with_relative_values(relative_weights, value_relative_embeddings)
-    output = output.transpose(2, 3).contiguous().view(b, d, t_t) # [b, n_h, t_t, d_k] -> [b, d, t_t]
-    return output, p_attn
-
-  def _matmul_with_relative_values(self, x, y):
-    """
-    x: [b, h, l, m]
-    y: [h or 1, m, d]
-    ret: [b, h, l, d]
-    """
-    ret = torch.matmul(x, y.unsqueeze(0))
-    return ret
-
-  def _matmul_with_relative_keys(self, x, y):
-    """
-    x: [b, h, l, d]
-    y: [h or 1, m, d]
-    ret: [b, h, l, m]
-    """
-    ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
-    return ret
-
-  def _get_relative_embeddings(self, relative_embeddings, length):
-    max_relative_position = 2 * self.window_size + 1
-    # Pad first before slice to avoid using cond ops.
-    pad_length = max(length - (self.window_size + 1), 0)
-    slice_start_position = max((self.window_size + 1) - length, 0)
-    slice_end_position = slice_start_position + 2 * length - 1
-    if pad_length > 0:
-      padded_relative_embeddings = F.pad(
-          relative_embeddings,
-          commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]))
-    else:
-      padded_relative_embeddings = relative_embeddings
-    used_relative_embeddings = padded_relative_embeddings[:,slice_start_position:slice_end_position]
-    return used_relative_embeddings
-
-  def _relative_position_to_absolute_position(self, x):
-    """
-    x: [b, h, l, 2*l-1]
-    ret: [b, h, l, l]
-    """
-    batch, heads, length, _ = x.size()
-    # Concat columns of pad to shift from relative to absolute indexing.
-    x = F.pad(x, commons.convert_pad_shape([[0,0],[0,0],[0,0],[0,1]]))
-
-    # Concat extra elements so to add up to shape (len+1, 2*len-1).
-    x_flat = x.view([batch, heads, length * 2 * length])
-    x_flat = F.pad(x_flat, commons.convert_pad_shape([[0,0],[0,0],[0,length-1]]))
-
-    # Reshape and slice out the padded elements.
-    x_final = x_flat.view([batch, heads, length+1, 2*length-1])[:, :, :length, length-1:]
-    return x_final
-
-  def _absolute_position_to_relative_position(self, x):
-    """
-    x: [b, h, l, l]
-    ret: [b, h, l, 2*l-1]
-    """
-    batch, heads, length, _ = x.size()
-    # padd along column
-    x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length-1]]))
-    x_flat = x.view([batch, heads, length**2 + length*(length -1)])
-    # add 0's in the beginning that will skew the elements after reshape
-    x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
-    x_final = x_flat.view([batch, heads, length, 2*length])[:,:,:,1:]
-    return x_final
-
-  def _attention_bias_proximal(self, length):
-    """Bias for self-attention to encourage attention to close positions.
-    Args:
-      length: an integer scalar.
-    Returns:
-      a Tensor with shape [1, 1, length, length]
-    """
-    r = torch.arange(length, dtype=torch.float32)
-    diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1)
-    return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0)
-
-
-class FFN(nn.Module):
-  def __init__(self, in_channels, out_channels, filter_channels, kernel_size, p_dropout=0., activation=None, causal=False):
-    super().__init__()
-    self.in_channels = in_channels
-    self.out_channels = out_channels
-    self.filter_channels = filter_channels
-    self.kernel_size = kernel_size
-    self.p_dropout = p_dropout
-    self.activation = activation
-    self.causal = causal
-
-    if causal:
-      self.padding = self._causal_padding
-    else:
-      self.padding = self._same_padding
-
-    self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size)
-    self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size)
-    self.drop = nn.Dropout(p_dropout)
-
-  def forward(self, x, x_mask):
-    x = self.conv_1(self.padding(x * x_mask))
-    if self.activation == "gelu":
-      x = x * torch.sigmoid(1.702 * x)
-    else:
-      x = torch.relu(x)
-    x = self.drop(x)
-    x = self.conv_2(self.padding(x * x_mask))
-    return x * x_mask
-  
-  def _causal_padding(self, x):
-    if self.kernel_size == 1:
-      return x
-    pad_l = self.kernel_size - 1
-    pad_r = 0
-    padding = [[0, 0], [0, 0], [pad_l, pad_r]]
-    x = F.pad(x, commons.convert_pad_shape(padding))
-    return x
-
-  def _same_padding(self, x):
-    if self.kernel_size == 1:
-      return x
-    pad_l = (self.kernel_size - 1) // 2
-    pad_r = self.kernel_size // 2
-    padding = [[0, 0], [0, 0], [pad_l, pad_r]]
-    x = F.pad(x, commons.convert_pad_shape(padding))
-    return x

src/tts_vits/commons.py

@@ -1,188 +0,0 @@
-import math
-import numpy as np
-import torch
-from torch import nn
-from torch.nn import functional as F
-
-def slice_pitch_segments(x, ids_str, segment_size=4):
-  ret = torch.zeros_like(x[:, :segment_size])
-  for i in range(x.size(0)):
-    idx_str = ids_str[i]
-    idx_end = idx_str + segment_size
-    ret[i] = x[i, idx_str:idx_end]
-  return ret
-
-def rand_slice_segments_with_pitch(x, pitch, x_lengths=None, segment_size=4):
-  b, d, t = x.size()
-  if x_lengths is None:
-    x_lengths = t
-  ids_str_max = x_lengths - segment_size + 1
-  ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
-  ret = slice_segments(x, ids_str, segment_size)
-  ret_pitch = slice_pitch_segments(pitch, ids_str, segment_size)
-  return ret, ret_pitch, ids_str
-
-def init_weights(m, mean=0.0, std=0.01):
-  classname = m.__class__.__name__
-  if classname.find("Conv") != -1:
-    m.weight.data.normal_(mean, std)
-
-
-def get_padding(kernel_size, dilation=1):
-  return int((kernel_size*dilation - dilation)/2)
-
-
-def convert_pad_shape(pad_shape):
-  l = pad_shape[::-1]
-  pad_shape = [item for sublist in l for item in sublist]
-  return pad_shape
-
-
-def intersperse(lst, item):
-  result = [item] * (len(lst) * 2 + 1)
-  result[1::2] = lst
-  return result
-
-
-def kl_divergence(m_p, logs_p, m_q, logs_q):
-  """KL(P||Q)"""
-  kl = (logs_q - logs_p) - 0.5
-  kl += 0.5 * (torch.exp(2. * logs_p) + ((m_p - m_q)**2)) * torch.exp(-2. * logs_q)
-  return kl
-
-
-def rand_gumbel(shape):
-  """Sample from the Gumbel distribution, protect from overflows."""
-  uniform_samples = torch.rand(shape) * 0.99998 + 0.00001
-  return -torch.log(-torch.log(uniform_samples))
-
-
-def rand_gumbel_like(x):
-  g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device)
-  return g
-
-
-def slice_segments(x, ids_str, segment_size=4):
-  ret = torch.zeros_like(x[:, :, :segment_size])
-  for i in range(x.size(0)):
-    idx_str = ids_str[i]
-    idx_end = idx_str + segment_size
-    ret[i] = x[i, :, idx_str:idx_end]
-  return ret
-
-
-def rand_slice_segments(x, x_lengths=None, segment_size=4):
-  b, d, t = x.size()
-  if x_lengths is None:
-    x_lengths = t
-  ids_str_max = x_lengths - segment_size + 1
-  ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
-  ret = slice_segments(x, ids_str, segment_size)
-  return ret, ids_str
-
-
-def rand_spec_segments(x, x_lengths=None, segment_size=4):
-  b, d, t = x.size()
-  if x_lengths is None:
-    x_lengths = t
-  ids_str_max = x_lengths - segment_size
-  ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
-  ret = slice_segments(x, ids_str, segment_size)
-  return ret, ids_str
-
-
-def get_timing_signal_1d(
-    length, channels, min_timescale=1.0, max_timescale=1.0e4):
-  position = torch.arange(length, dtype=torch.float)
-  num_timescales = channels // 2
-  log_timescale_increment = (
-      math.log(float(max_timescale) / float(min_timescale)) /
-      (num_timescales - 1))
-  inv_timescales = min_timescale * torch.exp(
-      torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment)
-  scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1)
-  signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0)
-  signal = F.pad(signal, [0, 0, 0, channels % 2])
-  signal = signal.view(1, channels, length)
-  return signal
-
-
-def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4):
-  b, channels, length = x.size()
-  signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
-  return x + signal.to(dtype=x.dtype, device=x.device)
-
-
-def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1):
-  b, channels, length = x.size()
-  signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
-  return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis)
-
-
-def subsequent_mask(length):
-  mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
-  return mask
-
-
-@torch.jit.script
-def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
-  n_channels_int = n_channels[0]
-  in_act = input_a + input_b
-  t_act = torch.tanh(in_act[:, :n_channels_int, :])
-  s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
-  acts = t_act * s_act
-  return acts
-
-
-def convert_pad_shape(pad_shape):
-  l = pad_shape[::-1]
-  pad_shape = [item for sublist in l for item in sublist]
-  return pad_shape
-
-
-def shift_1d(x):
-  x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1]
-  return x
-
-
-def sequence_mask(length, max_length=None):
-  if max_length is None:
-    max_length = length.max()
-  x = torch.arange(max_length, dtype=length.dtype, device=length.device)
-  return x.unsqueeze(0) < length.unsqueeze(1)
-
-
-def generate_path(duration, mask):
-  """
-  duration: [b, 1, t_x]
-  mask: [b, 1, t_y, t_x]
-  """
-  device = duration.device
-  
-  b, _, t_y, t_x = mask.shape
-  cum_duration = torch.cumsum(duration, -1)
-  
-  cum_duration_flat = cum_duration.view(b * t_x)
-  path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
-  path = path.view(b, t_x, t_y)
-  path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1]
-  path = path.unsqueeze(1).transpose(2,3) * mask
-  return path
-
-
-def clip_grad_value_(parameters, clip_value, norm_type=2):
-  if isinstance(parameters, torch.Tensor):
-    parameters = [parameters]
-  parameters = list(filter(lambda p: p.grad is not None, parameters))
-  norm_type = float(norm_type)
-  if clip_value is not None:
-    clip_value = float(clip_value)
-
-  total_norm = 0
-  for p in parameters:
-    param_norm = p.grad.data.norm(norm_type)
-    total_norm += param_norm.item() ** norm_type
-    if clip_value is not None:
-      p.grad.data.clamp_(min=-clip_value, max=clip_value)
-  total_norm = total_norm ** (1. / norm_type)
-  return total_norm

src/tts_vits/configs/config.json

@@ -1,90 +0,0 @@
-{
-  "train": {
-    "log_interval": 200,
-    "eval_interval": 1000,
-    "seed": 1234,
-    "epochs": 10000,
-    "learning_rate": 0.0001,
-    "betas": [
-      0.8,
-      0.99
-    ],
-    "eps": 1e-09,
-    "batch_size": 12,
-    "fp16_run": false,
-    "lr_decay": 0.999875,
-    "segment_size": 17920,
-    "init_lr_ratio": 1,
-    "warmup_epochs": 0,
-    "c_mel": 45,
-    "c_kl": 1.0,
-    "use_sr": true,
-    "max_speclen": 384,
-    "port": "8011"
-  },
-  "data": {
-    "training_files": "filelists/train.txt",
-    "validation_files": "filelists/val.txt",
-    "max_wav_value": 32768.0,
-    "sampling_rate": 32000,
-    "filter_length": 1280,
-    "hop_length": 320,
-    "win_length": 1280,
-    "n_mel_channels": 80,
-    "mel_fmin": 0.0,
-    "mel_fmax": null
-  },
-  "model": {
-    "inter_channels": 192,
-    "hidden_channels": 192,
-    "filter_channels": 768,
-    "n_heads": 2,
-    "n_layers": 6,
-    "kernel_size": 3,
-    "p_dropout": 0.1,
-    "resblock": "1",
-    "resblock_kernel_sizes": [
-      3,
-      7,
-      11
-    ],
-    "resblock_dilation_sizes": [
-      [
-        1,
-        3,
-        5
-      ],
-      [
-        1,
-        3,
-        5
-      ],
-      [
-        1,
-        3,
-        5
-      ]
-    ],
-    "upsample_rates": [
-      10,
-      8,
-      2,
-      2
-    ],
-    "upsample_initial_channel": 512,
-    "upsample_kernel_sizes": [
-      16,
-      16,
-      4,
-      4
-    ],
-    "n_layers_q": 3,
-    "use_spectral_norm": false,
-    "gin_channels": 256,
-    "ssl_dim": 256,
-    "n_speakers": 2
-  },
-  "spk": {
-    "haruhi": 0
-  }
-}

src/tts_vits/hubert/hubert_model.py

@@ -1,222 +0,0 @@
-import copy
-import random
-from typing import Optional, Tuple
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as t_func
-from torch.nn.modules.utils import consume_prefix_in_state_dict_if_present
-
-
-class Hubert(nn.Module):
-    def __init__(self, num_label_embeddings: int = 100, mask: bool = True):
-        super().__init__()
-        self._mask = mask
-        self.feature_extractor = FeatureExtractor()
-        self.feature_projection = FeatureProjection()
-        self.positional_embedding = PositionalConvEmbedding()
-        self.norm = nn.LayerNorm(768)
-        self.dropout = nn.Dropout(0.1)
-        self.encoder = TransformerEncoder(
-            nn.TransformerEncoderLayer(
-                768, 12, 3072, activation="gelu", batch_first=True
-            ),
-            12,
-        )
-        self.proj = nn.Linear(768, 256)
-
-        self.masked_spec_embed = nn.Parameter(torch.FloatTensor(768).uniform_())
-        self.label_embedding = nn.Embedding(num_label_embeddings, 256)
-
-    def mask(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
-        mask = None
-        if self.training and self._mask:
-            mask = _compute_mask((x.size(0), x.size(1)), 0.8, 10, x.device, 2)
-            x[mask] = self.masked_spec_embed.to(x.dtype)
-        return x, mask
-
-    def encode(
-            self, x: torch.Tensor, layer: Optional[int] = None
-    ) -> Tuple[torch.Tensor, torch.Tensor]:
-        x = self.feature_extractor(x)
-        x = self.feature_projection(x.transpose(1, 2))
-        x, mask = self.mask(x)
-        x = x + self.positional_embedding(x)
-        x = self.dropout(self.norm(x))
-        x = self.encoder(x, output_layer=layer)
-        return x, mask
-
-    def logits(self, x: torch.Tensor) -> torch.Tensor:
-        logits = torch.cosine_similarity(
-            x.unsqueeze(2),
-            self.label_embedding.weight.unsqueeze(0).unsqueeze(0),
-            dim=-1,
-        )
-        return logits / 0.1
-
-    def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
-        x, mask = self.encode(x)
-        x = self.proj(x)
-        logits = self.logits(x)
-        return logits, mask
-
-
-class HubertSoft(Hubert):
-    def __init__(self):
-        super().__init__()
-
-    @torch.inference_mode()
-    def units(self, wav: torch.Tensor) -> torch.Tensor:
-        wav = t_func.pad(wav, ((400 - 320) // 2, (400 - 320) // 2))
-        x, _ = self.encode(wav)
-        return self.proj(x)
-
-
-class FeatureExtractor(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.conv0 = nn.Conv1d(1, 512, 10, 5, bias=False)
-        self.norm0 = nn.GroupNorm(512, 512)
-        self.conv1 = nn.Conv1d(512, 512, 3, 2, bias=False)
-        self.conv2 = nn.Conv1d(512, 512, 3, 2, bias=False)
-        self.conv3 = nn.Conv1d(512, 512, 3, 2, bias=False)
-        self.conv4 = nn.Conv1d(512, 512, 3, 2, bias=False)
-        self.conv5 = nn.Conv1d(512, 512, 2, 2, bias=False)
-        self.conv6 = nn.Conv1d(512, 512, 2, 2, bias=False)
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        x = t_func.gelu(self.norm0(self.conv0(x)))
-        x = t_func.gelu(self.conv1(x))
-        x = t_func.gelu(self.conv2(x))
-        x = t_func.gelu(self.conv3(x))
-        x = t_func.gelu(self.conv4(x))
-        x = t_func.gelu(self.conv5(x))
-        x = t_func.gelu(self.conv6(x))
-        return x
-
-
-class FeatureProjection(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.norm = nn.LayerNorm(512)
-        self.projection = nn.Linear(512, 768)
-        self.dropout = nn.Dropout(0.1)
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        x = self.norm(x)
-        x = self.projection(x)
-        x = self.dropout(x)
-        return x
-
-
-class PositionalConvEmbedding(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.conv = nn.Conv1d(
-            768,
-            768,
-            kernel_size=128,
-            padding=128 // 2,
-            groups=16,
-        )
-        self.conv = nn.utils.weight_norm(self.conv, name="weight", dim=2)
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        x = self.conv(x.transpose(1, 2))
-        x = t_func.gelu(x[:, :, :-1])
-        return x.transpose(1, 2)
-
-
-class TransformerEncoder(nn.Module):
-    def __init__(
-            self, encoder_layer: nn.TransformerEncoderLayer, num_layers: int
-    ) -> None:
-        super(TransformerEncoder, self).__init__()
-        self.layers = nn.ModuleList(
-            [copy.deepcopy(encoder_layer) for _ in range(num_layers)]
-        )
-        self.num_layers = num_layers
-
-    def forward(
-            self,
-            src: torch.Tensor,
-            mask: torch.Tensor = None,
-            src_key_padding_mask: torch.Tensor = None,
-            output_layer: Optional[int] = None,
-    ) -> torch.Tensor:
-        output = src
-        for layer in self.layers[:output_layer]:
-            output = layer(
-                output, src_mask=mask, src_key_padding_mask=src_key_padding_mask
-            )
-        return output
-
-
-def _compute_mask(
-        shape: Tuple[int, int],
-        mask_prob: float,
-        mask_length: int,
-        device: torch.device,
-        min_masks: int = 0,
-) -> torch.Tensor:
-    batch_size, sequence_length = shape
-
-    if mask_length < 1:
-        raise ValueError("`mask_length` has to be bigger than 0.")
-
-    if mask_length > sequence_length:
-        raise ValueError(
-            f"`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: {mask_length} and `sequence_length`: {sequence_length}`"
-        )
-
-    # compute number of masked spans in batch
-    num_masked_spans = int(mask_prob * sequence_length / mask_length + random.random())
-    num_masked_spans = max(num_masked_spans, min_masks)
-
-    # make sure num masked indices <= sequence_length
-    if num_masked_spans * mask_length > sequence_length:
-        num_masked_spans = sequence_length // mask_length
-
-    # SpecAugment mask to fill
-    mask = torch.zeros((batch_size, sequence_length), device=device, dtype=torch.bool)
-
-    # uniform distribution to sample from, make sure that offset samples are < sequence_length
-    uniform_dist = torch.ones(
-        (batch_size, sequence_length - (mask_length - 1)), device=device
-    )
-
-    # get random indices to mask
-    mask_indices = torch.multinomial(uniform_dist, num_masked_spans)
-
-    # expand masked indices to masked spans
-    mask_indices = (
-        mask_indices.unsqueeze(dim=-1)
-        .expand((batch_size, num_masked_spans, mask_length))
-        .reshape(batch_size, num_masked_spans * mask_length)
-    )
-    offsets = (
-        torch.arange(mask_length, device=device)[None, None, :]
-        .expand((batch_size, num_masked_spans, mask_length))
-        .reshape(batch_size, num_masked_spans * mask_length)
-    )
-    mask_idxs = mask_indices + offsets
-
-    # scatter indices to mask
-    mask = mask.scatter(1, mask_idxs, True)
-
-    return mask
-
-
-def hubert_soft(
-        path: str,
-) -> HubertSoft:
-    r"""HuBERT-Soft from `"A Comparison of Discrete and Soft Speech Units for Improved Voice Conversion"`.
-    Args:
-        path (str): path of a pretrained model
-    """
-    hubert = HubertSoft()
-    checkpoint = torch.load(path)
-    consume_prefix_in_state_dict_if_present(checkpoint, "module.")
-    hubert.load_state_dict(checkpoint)
-    hubert.eval()
-    return hubert

src/tts_vits/hubert/hubert_model_onnx.py

@@ -1,217 +0,0 @@
-import copy
-import random
-from typing import Optional, Tuple
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as t_func
-from torch.nn.modules.utils import consume_prefix_in_state_dict_if_present
-
-
-class Hubert(nn.Module):
-    def __init__(self, num_label_embeddings: int = 100, mask: bool = True):
-        super().__init__()
-        self._mask = mask
-        self.feature_extractor = FeatureExtractor()
-        self.feature_projection = FeatureProjection()
-        self.positional_embedding = PositionalConvEmbedding()
-        self.norm = nn.LayerNorm(768)
-        self.dropout = nn.Dropout(0.1)
-        self.encoder = TransformerEncoder(
-            nn.TransformerEncoderLayer(
-                768, 12, 3072, activation="gelu", batch_first=True
-            ),
-            12,
-        )
-        self.proj = nn.Linear(768, 256)
-
-        self.masked_spec_embed = nn.Parameter(torch.FloatTensor(768).uniform_())
-        self.label_embedding = nn.Embedding(num_label_embeddings, 256)
-
-    def mask(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
-        mask = None
-        if self.training and self._mask:
-            mask = _compute_mask((x.size(0), x.size(1)), 0.8, 10, x.device, 2)
-            x[mask] = self.masked_spec_embed.to(x.dtype)
-        return x, mask
-
-    def encode(
-            self, x: torch.Tensor, layer: Optional[int] = None
-    ) -> Tuple[torch.Tensor, torch.Tensor]:
-        x = self.feature_extractor(x)
-        x = self.feature_projection(x.transpose(1, 2))
-        x, mask = self.mask(x)
-        x = x + self.positional_embedding(x)
-        x = self.dropout(self.norm(x))
-        x = self.encoder(x, output_layer=layer)
-        return x, mask
-
-    def logits(self, x: torch.Tensor) -> torch.Tensor:
-        logits = torch.cosine_similarity(
-            x.unsqueeze(2),
-            self.label_embedding.weight.unsqueeze(0).unsqueeze(0),
-            dim=-1,
-        )
-        return logits / 0.1
-
-
-class HubertSoft(Hubert):
-    def __init__(self):
-        super().__init__()
-
-    def units(self, wav: torch.Tensor) -> torch.Tensor:
-        wav = t_func.pad(wav, ((400 - 320) // 2, (400 - 320) // 2))
-        x, _ = self.encode(wav)
-        return self.proj(x)
-
-    def forward(self, x):
-        return self.units(x)
-
-class FeatureExtractor(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.conv0 = nn.Conv1d(1, 512, 10, 5, bias=False)
-        self.norm0 = nn.GroupNorm(512, 512)
-        self.conv1 = nn.Conv1d(512, 512, 3, 2, bias=False)
-        self.conv2 = nn.Conv1d(512, 512, 3, 2, bias=False)
-        self.conv3 = nn.Conv1d(512, 512, 3, 2, bias=False)
-        self.conv4 = nn.Conv1d(512, 512, 3, 2, bias=False)
-        self.conv5 = nn.Conv1d(512, 512, 2, 2, bias=False)
-        self.conv6 = nn.Conv1d(512, 512, 2, 2, bias=False)
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        x = t_func.gelu(self.norm0(self.conv0(x)))
-        x = t_func.gelu(self.conv1(x))
-        x = t_func.gelu(self.conv2(x))
-        x = t_func.gelu(self.conv3(x))
-        x = t_func.gelu(self.conv4(x))
-        x = t_func.gelu(self.conv5(x))
-        x = t_func.gelu(self.conv6(x))
-        return x
-
-
-class FeatureProjection(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.norm = nn.LayerNorm(512)
-        self.projection = nn.Linear(512, 768)
-        self.dropout = nn.Dropout(0.1)
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        x = self.norm(x)
-        x = self.projection(x)
-        x = self.dropout(x)
-        return x
-
-
-class PositionalConvEmbedding(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.conv = nn.Conv1d(
-            768,
-            768,
-            kernel_size=128,
-            padding=128 // 2,
-            groups=16,
-        )
-        self.conv = nn.utils.weight_norm(self.conv, name="weight", dim=2)
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        x = self.conv(x.transpose(1, 2))
-        x = t_func.gelu(x[:, :, :-1])
-        return x.transpose(1, 2)
-
-
-class TransformerEncoder(nn.Module):
-    def __init__(
-            self, encoder_layer: nn.TransformerEncoderLayer, num_layers: int
-    ) -> None:
-        super(TransformerEncoder, self).__init__()
-        self.layers = nn.ModuleList(
-            [copy.deepcopy(encoder_layer) for _ in range(num_layers)]
-        )
-        self.num_layers = num_layers
-
-    def forward(
-            self,
-            src: torch.Tensor,
-            mask: torch.Tensor = None,
-            src_key_padding_mask: torch.Tensor = None,
-            output_layer: Optional[int] = None,
-    ) -> torch.Tensor:
-        output = src
-        for layer in self.layers[:output_layer]:
-            output = layer(
-                output, src_mask=mask, src_key_padding_mask=src_key_padding_mask
-            )
-        return output
-
-
-def _compute_mask(
-        shape: Tuple[int, int],
-        mask_prob: float,
-        mask_length: int,
-        device: torch.device,
-        min_masks: int = 0,
-) -> torch.Tensor:
-    batch_size, sequence_length = shape
-
-    if mask_length < 1:
-        raise ValueError("`mask_length` has to be bigger than 0.")
-
-    if mask_length > sequence_length:
-        raise ValueError(
-            f"`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: {mask_length} and `sequence_length`: {sequence_length}`"
-        )
-
-    # compute number of masked spans in batch
-    num_masked_spans = int(mask_prob * sequence_length / mask_length + random.random())
-    num_masked_spans = max(num_masked_spans, min_masks)
-
-    # make sure num masked indices <= sequence_length
-    if num_masked_spans * mask_length > sequence_length:
-        num_masked_spans = sequence_length // mask_length
-
-    # SpecAugment mask to fill
-    mask = torch.zeros((batch_size, sequence_length), device=device, dtype=torch.bool)
-
-    # uniform distribution to sample from, make sure that offset samples are < sequence_length
-    uniform_dist = torch.ones(
-        (batch_size, sequence_length - (mask_length - 1)), device=device
-    )
-
-    # get random indices to mask
-    mask_indices = torch.multinomial(uniform_dist, num_masked_spans)
-
-    # expand masked indices to masked spans
-    mask_indices = (
-        mask_indices.unsqueeze(dim=-1)
-        .expand((batch_size, num_masked_spans, mask_length))
-        .reshape(batch_size, num_masked_spans * mask_length)
-    )
-    offsets = (
-        torch.arange(mask_length, device=device)[None, None, :]
-        .expand((batch_size, num_masked_spans, mask_length))
-        .reshape(batch_size, num_masked_spans * mask_length)
-    )
-    mask_idxs = mask_indices + offsets
-
-    # scatter indices to mask
-    mask = mask.scatter(1, mask_idxs, True)
-
-    return mask
-
-
-def hubert_soft(
-        path: str,
-) -> HubertSoft:
-    r"""HuBERT-Soft from `"A Comparison of Discrete and Soft Speech Units for Improved Voice Conversion"`.
-    Args:
-        path (str): path of a pretrained model
-    """
-    hubert = HubertSoft()
-    checkpoint = torch.load(path)
-    consume_prefix_in_state_dict_if_present(checkpoint, "module.")
-    hubert.load_state_dict(checkpoint)
-    hubert.eval()
-    return hubert

src/tts_vits/inference/chunks_temp.json

@@ -1 +0,0 @@
-{"info": "temp_dict"}

src/tts_vits/inference/infer_tool.py

@@ -1,326 +0,0 @@
-import hashlib
-import json
-import logging
-import os
-import time
-from pathlib import Path
-
-import librosa
-import maad
-import numpy as np
-# import onnxruntime
-import parselmouth
-import soundfile
-import torch
-import torchaudio
-
-from hubert import hubert_model
-import utils
-from models import SynthesizerTrn
-
-logging.getLogger('matplotlib').setLevel(logging.WARNING)
-
-
-def read_temp(file_name):
-    if not os.path.exists(file_name):
-        with open(file_name, "w") as f:
-            f.write(json.dumps({"info": "temp_dict"}))
-        return {}
-    else:
-        try:
-            with open(file_name, "r") as f:
-                data = f.read()
-            data_dict = json.loads(data)
-            if os.path.getsize(file_name) > 50 * 1024 * 1024:
-                f_name = file_name.split("/")[-1]
-                print(f"clean {f_name}")
-                for wav_hash in list(data_dict.keys()):
-                    if int(time.time()) - int(data_dict[wav_hash]["time"]) > 14 * 24 * 3600:
-                        del data_dict[wav_hash]
-        except Exception as e:
-            print(e)
-            print(f"{file_name} error,auto rebuild file")
-            data_dict = {"info": "temp_dict"}
-        return data_dict
-
-
-def write_temp(file_name, data):
-    with open(file_name, "w") as f:
-        f.write(json.dumps(data))
-
-
-def timeit(func):
-    def run(*args, **kwargs):
-        t = time.time()
-        res = func(*args, **kwargs)
-        print('executing \'%s\' costed %.3fs' % (func.__name__, time.time() - t))
-        return res
-
-    return run
-
-
-def format_wav(audio_path):
-    if Path(audio_path).suffix == '.wav':
-        return
-    raw_audio, raw_sample_rate = librosa.load(audio_path, mono=True, sr=None)
-    soundfile.write(Path(audio_path).with_suffix(".wav"), raw_audio, raw_sample_rate)
-
-
-def get_end_file(dir_path, end):
-    file_lists = []
-    for root, dirs, files in os.walk(dir_path):
-        files = [f for f in files if f[0] != '.']
-        dirs[:] = [d for d in dirs if d[0] != '.']
-        for f_file in files:
-            if f_file.endswith(end):
-                file_lists.append(os.path.join(root, f_file).replace("\\", "/"))
-    return file_lists
-
-
-def get_md5(content):
-    return hashlib.new("md5", content).hexdigest()
-
-
-def resize2d_f0(x, target_len):
-    source = np.array(x)
-    source[source < 0.001] = np.nan
-    target = np.interp(np.arange(0, len(source) * target_len, len(source)) / target_len, np.arange(0, len(source)),
-                       source)
-    res = np.nan_to_num(target)
-    return res
-
-def get_f0(x, p_len,f0_up_key=0):
-
-    time_step = 160 / 16000 * 1000
-    f0_min = 50
-    f0_max = 1100
-    f0_mel_min = 1127 * np.log(1 + f0_min / 700)
-    f0_mel_max = 1127 * np.log(1 + f0_max / 700)
-
-    f0 = parselmouth.Sound(x, 16000).to_pitch_ac(
-        time_step=time_step / 1000, voicing_threshold=0.6,
-        pitch_floor=f0_min, pitch_ceiling=f0_max).selected_array['frequency']
-
-    pad_size=(p_len - len(f0) + 1) // 2
-    if(pad_size>0 or p_len - len(f0) - pad_size>0):
-        f0 = np.pad(f0,[[pad_size,p_len - len(f0) - pad_size]], mode='constant')
-
-    f0 *= pow(2, f0_up_key / 12)
-    f0_mel = 1127 * np.log(1 + f0 / 700)
-    f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / (f0_mel_max - f0_mel_min) + 1
-    f0_mel[f0_mel <= 1] = 1
-    f0_mel[f0_mel > 255] = 255
-    f0_coarse = np.rint(f0_mel).astype(np.int)
-    return f0_coarse, f0
-
-def clean_pitch(input_pitch):
-    num_nan = np.sum(input_pitch == 1)
-    if num_nan / len(input_pitch) > 0.9:
-        input_pitch[input_pitch != 1] = 1
-    return input_pitch
-
-
-def plt_pitch(input_pitch):
-    input_pitch = input_pitch.astype(float)
-    input_pitch[input_pitch == 1] = np.nan
-    return input_pitch
-
-
-def f0_to_pitch(ff):
-    f0_pitch = 69 + 12 * np.log2(ff / 440)
-    return f0_pitch
-
-
-def fill_a_to_b(a, b):
-    if len(a) < len(b):
-        for _ in range(0, len(b) - len(a)):
-            a.append(a[0])
-
-
-def mkdir(paths: list):
-    for path in paths:
-        if not os.path.exists(path):
-            os.mkdir(path)
-
-
-class Svc(object):
-    def __init__(self, net_g_path, config_path, hubert_path="hubert/hubert-soft-0d54a1f4.pt",
-                 onnx=False):
-        self.onnx = onnx
-        self.net_g_path = net_g_path
-        self.hubert_path = hubert_path
-        self.dev = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        self.net_g_ms = None
-        self.hps_ms = utils.get_hparams_from_file(config_path)
-        self.target_sample = self.hps_ms.data.sampling_rate
-        self.hop_size = self.hps_ms.data.hop_length
-        self.speakers = {}
-        for spk, sid in self.hps_ms.spk.items():
-            self.speakers[sid] = spk
-        self.spk2id = self.hps_ms.spk
-        # 加载hubert
-        self.hubert_soft = hubert_model.hubert_soft(hubert_path)
-        if torch.cuda.is_available():
-            self.hubert_soft = self.hubert_soft.cuda()
-        self.load_model()
-
-    def load_model(self):
-        # 获取模型配置
-        if self.onnx:
-            raise NotImplementedError
-            # self.net_g_ms = SynthesizerTrnForONNX(
-            #     178,
-            #     self.hps_ms.data.filter_length // 2 + 1,
-            #     self.hps_ms.train.segment_size // self.hps_ms.data.hop_length,
-            #     n_speakers=self.hps_ms.data.n_speakers,
-            #     **self.hps_ms.model)
-            # _ = utils.load_checkpoint(self.net_g_path, self.net_g_ms, None)
-        else:
-            self.net_g_ms = SynthesizerTrn(
-                self.hps_ms.data.filter_length // 2 + 1,
-                self.hps_ms.train.segment_size // self.hps_ms.data.hop_length,
-                **self.hps_ms.model)
-            _ = utils.load_checkpoint(self.net_g_path, self.net_g_ms, None)
-        if "half" in self.net_g_path and torch.cuda.is_available():
-            _ = self.net_g_ms.half().eval().to(self.dev)
-        else:
-            _ = self.net_g_ms.eval().to(self.dev)
-
-    def get_units(self, source, sr):
-
-        source = source.unsqueeze(0).to(self.dev)
-        with torch.inference_mode():
-            start = time.time()
-            units = self.hubert_soft.units(source)
-            use_time = time.time() - start
-            print("hubert use time:{}".format(use_time))
-            return units
-
-
-    def get_unit_pitch(self, in_path, tran):
-        source, sr = torchaudio.load(in_path)
-        source = torchaudio.functional.resample(source, sr, 16000)
-        if len(source.shape) == 2 and source.shape[1] >= 2:
-            source = torch.mean(source, dim=0).unsqueeze(0)
-        soft = self.get_units(source, sr).squeeze(0).cpu().numpy()
-        f0_coarse, f0 = get_f0(source.cpu().numpy()[0], soft.shape[0]*2, tran)
-        return soft, f0
-
-    def infer(self, speaker_id, tran, raw_path):
-        if type(speaker_id) == str:
-            speaker_id = self.spk2id[speaker_id]
-        sid = torch.LongTensor([int(speaker_id)]).to(self.dev).unsqueeze(0)
-        soft, pitch = self.get_unit_pitch(raw_path, tran)
-        f0 = torch.FloatTensor(clean_pitch(pitch)).unsqueeze(0).to(self.dev)
-        if "half" in self.net_g_path and torch.cuda.is_available():
-            stn_tst = torch.HalfTensor(soft)
-        else:
-            stn_tst = torch.FloatTensor(soft)
-        with torch.no_grad():
-            x_tst = stn_tst.unsqueeze(0).to(self.dev)
-            start = time.time()
-            x_tst = torch.repeat_interleave(x_tst, repeats=2, dim=1).transpose(1, 2)
-            audio = self.net_g_ms.infer(x_tst, f0=f0, g=sid)[0,0].data.float()
-            use_time = time.time() - start
-            print("vits use time:{}".format(use_time))
-        return audio, audio.shape[-1]
-
-
-# class SvcONNXInferModel(object):
-#     def __init__(self, hubert_onnx, vits_onnx, config_path):
-#         self.config_path = config_path
-#         self.vits_onnx = vits_onnx
-#         self.hubert_onnx = hubert_onnx
-#         self.hubert_onnx_session = onnxruntime.InferenceSession(hubert_onnx, providers=['CUDAExecutionProvider', ])
-#         self.inspect_onnx(self.hubert_onnx_session)
-#         self.vits_onnx_session = onnxruntime.InferenceSession(vits_onnx, providers=['CUDAExecutionProvider', ])
-#         self.inspect_onnx(self.vits_onnx_session)
-#         self.hps_ms = utils.get_hparams_from_file(self.config_path)
-#         self.target_sample = self.hps_ms.data.sampling_rate
-#         self.feature_input = FeatureInput(self.hps_ms.data.sampling_rate, self.hps_ms.data.hop_length)
-#
-#     @staticmethod
-#     def inspect_onnx(session):
-#         for i in session.get_inputs():
-#             print("name:{}\tshape:{}\tdtype:{}".format(i.name, i.shape, i.type))
-#         for i in session.get_outputs():
-#             print("name:{}\tshape:{}\tdtype:{}".format(i.name, i.shape, i.type))
-#
-#     def infer(self, speaker_id, tran, raw_path):
-#         sid = np.array([int(speaker_id)], dtype=np.int64)
-#         soft, pitch = self.get_unit_pitch(raw_path, tran)
-#         pitch = np.expand_dims(pitch, axis=0).astype(np.int64)
-#         stn_tst = soft
-#         x_tst = np.expand_dims(stn_tst, axis=0)
-#         x_tst_lengths = np.array([stn_tst.shape[0]], dtype=np.int64)
-#         # 使用ONNX Runtime进行推理
-#         start = time.time()
-#         audio = self.vits_onnx_session.run(output_names=["audio"],
-#                                            input_feed={
-#                                                "hidden_unit": x_tst,
-#                                                "lengths": x_tst_lengths,
-#                                                "pitch": pitch,
-#                                                "sid": sid,
-#                                            })[0][0, 0]
-#         use_time = time.time() - start
-#         print("vits_onnx_session.run time:{}".format(use_time))
-#         audio = torch.from_numpy(audio)
-#         return audio, audio.shape[-1]
-#
-#     def get_units(self, source, sr):
-#         source = torchaudio.functional.resample(source, sr, 16000)
-#         if len(source.shape) == 2 and source.shape[1] >= 2:
-#             source = torch.mean(source, dim=0).unsqueeze(0)
-#         source = source.unsqueeze(0)
-#         # 使用ONNX Runtime进行推理
-#         start = time.time()
-#         units = self.hubert_onnx_session.run(output_names=["embed"],
-#                                              input_feed={"source": source.numpy()})[0]
-#         use_time = time.time() - start
-#         print("hubert_onnx_session.run time:{}".format(use_time))
-#         return units
-#
-#     def transcribe(self, source, sr, length, transform):
-#         feature_pit = self.feature_input.compute_f0(source, sr)
-#         feature_pit = feature_pit * 2 ** (transform / 12)
-#         feature_pit = resize2d_f0(feature_pit, length)
-#         coarse_pit = self.feature_input.coarse_f0(feature_pit)
-#         return coarse_pit
-#
-#     def get_unit_pitch(self, in_path, tran):
-#         source, sr = torchaudio.load(in_path)
-#         soft = self.get_units(source, sr).squeeze(0)
-#         input_pitch = self.transcribe(source.numpy()[0], sr, soft.shape[0], tran)
-#         return soft, input_pitch
-
-
-class RealTimeVC:
-    def __init__(self):
-        self.last_chunk = None
-        self.last_o = None
-        self.chunk_len = 16000  # 区块长度
-        self.pre_len = 3840  # 交叉淡化长度，640的倍数
-
-    """输入输出都是1维numpy 音频波形数组"""
-
-    def process(self, svc_model, speaker_id, f_pitch_change, input_wav_path):
-        audio, sr = torchaudio.load(input_wav_path)
-        audio = audio.cpu().numpy()[0]
-        temp_wav = io.BytesIO()
-        if self.last_chunk is None:
-            input_wav_path.seek(0)
-            audio, sr = svc_model.infer(speaker_id, f_pitch_change, input_wav_path)
-            audio = audio.cpu().numpy()
-            self.last_chunk = audio[-self.pre_len:]
-            self.last_o = audio
-            return audio[-self.chunk_len:]
-        else:
-            audio = np.concatenate([self.last_chunk, audio])
-            soundfile.write(temp_wav, audio, sr, format="wav")
-            temp_wav.seek(0)
-            audio, sr = svc_model.infer(speaker_id, f_pitch_change, temp_wav)
-            audio = audio.cpu().numpy()
-            ret = maad.util.crossfade(self.last_o, audio, self.pre_len)
-            self.last_chunk = audio[-self.pre_len:]
-            self.last_o = audio
-            return ret[self.chunk_len:2 * self.chunk_len]

src/tts_vits/inference/infer_tool_grad.py

@@ -1,160 +0,0 @@
-import hashlib
-import json
-import logging
-import os
-import time
-from pathlib import Path
-import io
-import librosa
-import maad
-import numpy as np
-from inference import slicer
-import parselmouth
-import soundfile
-import torch
-import torchaudio
-
-from hubert import hubert_model
-import utils
-from models import SynthesizerTrn
-logging.getLogger('numba').setLevel(logging.WARNING)
-logging.getLogger('matplotlib').setLevel(logging.WARNING)
-
-def resize2d_f0(x, target_len):
-    source = np.array(x)
-    source[source < 0.001] = np.nan
-    target = np.interp(np.arange(0, len(source) * target_len, len(source)) / target_len, np.arange(0, len(source)),
-                       source)
-    res = np.nan_to_num(target)
-    return res
-
-def get_f0(x, p_len,f0_up_key=0):
-
-    time_step = 160 / 16000 * 1000
-    f0_min = 50
-    f0_max = 1100
-    f0_mel_min = 1127 * np.log(1 + f0_min / 700)
-    f0_mel_max = 1127 * np.log(1 + f0_max / 700)
-
-    f0 = parselmouth.Sound(x, 16000).to_pitch_ac(
-        time_step=time_step / 1000, voicing_threshold=0.6,
-        pitch_floor=f0_min, pitch_ceiling=f0_max).selected_array['frequency']
-
-    pad_size=(p_len - len(f0) + 1) // 2
-    if(pad_size>0 or p_len - len(f0) - pad_size>0):
-        f0 = np.pad(f0,[[pad_size,p_len - len(f0) - pad_size]], mode='constant')
-
-    f0 *= pow(2, f0_up_key / 12)
-    f0_mel = 1127 * np.log(1 + f0 / 700)
-    f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / (f0_mel_max - f0_mel_min) + 1
-    f0_mel[f0_mel <= 1] = 1
-    f0_mel[f0_mel > 255] = 255
-    f0_coarse = np.rint(f0_mel).astype(np.int)
-    return f0_coarse, f0
-
-def clean_pitch(input_pitch):
-    num_nan = np.sum(input_pitch == 1)
-    if num_nan / len(input_pitch) > 0.9:
-        input_pitch[input_pitch != 1] = 1
-    return input_pitch
-
-
-def plt_pitch(input_pitch):
-    input_pitch = input_pitch.astype(float)
-    input_pitch[input_pitch == 1] = np.nan
-    return input_pitch
-
-
-def f0_to_pitch(ff):
-    f0_pitch = 69 + 12 * np.log2(ff / 440)
-    return f0_pitch
-
-
-def fill_a_to_b(a, b):
-    if len(a) < len(b):
-        for _ in range(0, len(b) - len(a)):
-            a.append(a[0])
-
-
-def mkdir(paths: list):
-    for path in paths:
-        if not os.path.exists(path):
-            os.mkdir(path)
-
-
-class VitsSvc(object):
-    def __init__(self):
-        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        self.SVCVITS = None
-        self.hps = None
-        self.speakers = None
-        self.hubert_soft = hubert_model.hubert_soft("hubert/model.pt")
-
-    def set_device(self, device):
-        self.device = torch.device(device)
-        self.hubert_soft.to(self.device)
-        if self.SVCVITS != None:
-            self.SVCVITS.to(self.device)
-
-    def loadCheckpoint(self, path):
-        self.hps = utils.get_hparams_from_file(f"checkpoints/{path}/config.json")
-        self.SVCVITS = SynthesizerTrn(
-            self.hps.data.filter_length // 2 + 1,
-            self.hps.train.segment_size // self.hps.data.hop_length,
-            **self.hps.model)
-        _ = utils.load_checkpoint(f"checkpoints/{path}/model.pth", self.SVCVITS, None)
-        _ = self.SVCVITS.eval().to(self.device)
-        self.speakers = self.hps.spk
-
-    def get_units(self, source, sr):
-        source = source.unsqueeze(0).to(self.device)
-        with torch.inference_mode():
-            units = self.hubert_soft.units(source)
-            return units
-
-
-    def get_unit_pitch(self, in_path, tran):
-        source, sr = torchaudio.load(in_path)
-        source = torchaudio.functional.resample(source, sr, 16000)
-        if len(source.shape) == 2 and source.shape[1] >= 2:
-            source = torch.mean(source, dim=0).unsqueeze(0)
-        soft = self.get_units(source, sr).squeeze(0).cpu().numpy()
-        f0_coarse, f0 = get_f0(source.cpu().numpy()[0], soft.shape[0]*2, tran)
-        return soft, f0
-
-    def infer(self, speaker_id, tran, raw_path):
-        speaker_id = self.speakers[speaker_id]
-        sid = torch.LongTensor([int(speaker_id)]).to(self.device).unsqueeze(0)
-        soft, pitch = self.get_unit_pitch(raw_path, tran)
-        f0 = torch.FloatTensor(clean_pitch(pitch)).unsqueeze(0).to(self.device)
-        stn_tst = torch.FloatTensor(soft)
-        with torch.no_grad():
-            x_tst = stn_tst.unsqueeze(0).to(self.device)
-            x_tst = torch.repeat_interleave(x_tst, repeats=2, dim=1).transpose(1, 2)
-            audio = self.SVCVITS.infer(x_tst, f0=f0, g=sid)[0,0].data.float()
-        return audio, audio.shape[-1]
-
-    def inference(self,srcaudio,chara,tran,slice_db):
-        sampling_rate, audio = srcaudio
-        audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32)
-        if len(audio.shape) > 1:
-            audio = librosa.to_mono(audio.transpose(1, 0))
-        if sampling_rate != 16000:
-            audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=16000)
-        soundfile.write("tmpwav.wav", audio, 16000, format="wav")
-        chunks = slicer.cut("tmpwav.wav", db_thresh=slice_db)
-        audio_data, audio_sr = slicer.chunks2audio("tmpwav.wav", chunks)
-        audio = []
-        for (slice_tag, data) in audio_data:
-            length = int(np.ceil(len(data) / audio_sr * self.hps.data.sampling_rate))
-            raw_path = io.BytesIO()
-            soundfile.write(raw_path, data, audio_sr, format="wav")
-            raw_path.seek(0)
-            if slice_tag:
-                _audio = np.zeros(length)
-            else:
-                out_audio, out_sr = self.infer(chara, tran, raw_path)
-                _audio = out_audio.cpu().numpy()
-            audio.extend(list(_audio))
-        audio = (np.array(audio) * 32768.0).astype('int16')
-        return (self.hps.data.sampling_rate,audio)

src/tts_vits/inference/slicer.py

@@ -1,145 +0,0 @@
-import librosa
-import torch
-import torchaudio
-
-
-class Slicer:
-    def __init__(self,
-                 sr: int,
-                 threshold: float = -40.,
-                 min_length: int = 5000,
-                 min_interval: int = 300,
-                 hop_size: int = 20,
-                 max_sil_kept: int = 5000):
-        if not min_length >= min_interval >= hop_size:
-            raise ValueError('The following condition must be satisfied: min_length >= min_interval >= hop_size')
-        if not max_sil_kept >= hop_size:
-            raise ValueError('The following condition must be satisfied: max_sil_kept >= hop_size')
-        min_interval = sr * min_interval / 1000
-        self.threshold = 10 ** (threshold / 20.)
-        self.hop_size = round(sr * hop_size / 1000)
-        self.win_size = min(round(min_interval), 4 * self.hop_size)
-        self.min_length = round(sr * min_length / 1000 / self.hop_size)
-        self.min_interval = round(min_interval / self.hop_size)
-        self.max_sil_kept = round(sr * max_sil_kept / 1000 / self.hop_size)
-
-    def _apply_slice(self, waveform, begin, end):
-        if len(waveform.shape) > 1:
-            return waveform[:, begin * self.hop_size: min(waveform.shape[1], end * self.hop_size)]
-        else:
-            return waveform[begin * self.hop_size: min(waveform.shape[0], end * self.hop_size)]
-
-    # @timeit
-    def slice(self, waveform):
-        if len(waveform.shape) > 1:
-            samples = librosa.to_mono(waveform)
-        else:
-            samples = waveform
-        if samples.shape[0] <= self.min_length:
-            return {"0": {"slice": False, "split_time": f"0,{len(waveform)}"}}
-        rms_list = librosa.feature.rms(y=samples, frame_length=self.win_size, hop_length=self.hop_size).squeeze(0)
-        sil_tags = []
-        silence_start = None
-        clip_start = 0
-        for i, rms in enumerate(rms_list):
-            # Keep looping while frame is silent.
-            if rms < self.threshold:
-                # Record start of silent frames.
-                if silence_start is None:
-                    silence_start = i
-                continue
-            # Keep looping while frame is not silent and silence start has not been recorded.
-            if silence_start is None:
-                continue
-            # Clear recorded silence start if interval is not enough or clip is too short
-            is_leading_silence = silence_start == 0 and i > self.max_sil_kept
-            need_slice_middle = i - silence_start >= self.min_interval and i - clip_start >= self.min_length
-            if not is_leading_silence and not need_slice_middle:
-                silence_start = None
-                continue
-            # Need slicing. Record the range of silent frames to be removed.
-            if i - silence_start <= self.max_sil_kept:
-                pos = rms_list[silence_start: i + 1].argmin() + silence_start
-                if silence_start == 0:
-                    sil_tags.append((0, pos))
-                else:
-                    sil_tags.append((pos, pos))
-                clip_start = pos
-            elif i - silence_start <= self.max_sil_kept * 2:
-                pos = rms_list[i - self.max_sil_kept: silence_start + self.max_sil_kept + 1].argmin()
-                pos += i - self.max_sil_kept
-                pos_l = rms_list[silence_start: silence_start + self.max_sil_kept + 1].argmin() + silence_start
-                pos_r = rms_list[i - self.max_sil_kept: i + 1].argmin() + i - self.max_sil_kept
-                if silence_start == 0:
-                    sil_tags.append((0, pos_r))
-                    clip_start = pos_r
-                else:
-                    sil_tags.append((min(pos_l, pos), max(pos_r, pos)))
-                    clip_start = max(pos_r, pos)
-            else:
-                pos_l = rms_list[silence_start: silence_start + self.max_sil_kept + 1].argmin() + silence_start
-                pos_r = rms_list[i - self.max_sil_kept: i + 1].argmin() + i - self.max_sil_kept
-                if silence_start == 0:
-                    sil_tags.append((0, pos_r))
-                else:
-                    sil_tags.append((pos_l, pos_r))
-                clip_start = pos_r
-            silence_start = None
-        # Deal with trailing silence.
-        total_frames = rms_list.shape[0]
-        if silence_start is not None and total_frames - silence_start >= self.min_interval:
-            silence_end = min(total_frames, silence_start + self.max_sil_kept)
-            pos = rms_list[silence_start: silence_end + 1].argmin() + silence_start
-            sil_tags.append((pos, total_frames + 1))
-        # Apply and return slices.
-        if len(sil_tags) == 0:
-            return {"0": {"slice": False, "split_time": f"0,{len(waveform)}"}}
-        else:
-            chunks = []
-            # 第一段静音并非从头开始，补上有声片段
-            if sil_tags[0][0]:
-                chunks.append(
-                    {"slice": False, "split_time": f"0,{min(waveform.shape[0], sil_tags[0][0] * self.hop_size)}"})
-            for i in range(0, len(sil_tags)):
-                # 标识有声片段（跳过第一段）
-                if i:
-                    chunks.append({"slice": False,
-                                   "split_time": f"{sil_tags[i - 1][1] * self.hop_size},{min(waveform.shape[0], sil_tags[i][0] * self.hop_size)}"})
-                # 标识所有静音片段
-                chunks.append({"slice": True,
-                               "split_time": f"{sil_tags[i][0] * self.hop_size},{min(waveform.shape[0], sil_tags[i][1] * self.hop_size)}"})
-            # 最后一段静音并非结尾，补上结尾片段
-            if sil_tags[-1][1] * self.hop_size < len(waveform):
-                chunks.append({"slice": False, "split_time": f"{sil_tags[-1][1] * self.hop_size},{len(waveform)}"})
-            chunk_dict = {}
-            for i in range(len(chunks)):
-                chunk_dict[str(i)] = chunks[i]
-            return chunk_dict
-
-
-def cut(audio_path, db_thresh=-30, min_len=5000):
-    audio, sr = librosa.load(audio_path, sr=None)
-    slicer = Slicer(
-        sr=sr,
-        threshold=db_thresh,
-        min_length=min_len
-    )
-    chunks = slicer.slice(audio)
-    return chunks
-
-
-def chunks2audio(audio_path, chunks):
-    chunks = dict(chunks)
-    audio, sr = torchaudio.load(audio_path)
-    # audio, sr = librosa.load(audio_path, sr=None)
-
-    if len(audio.shape) == 2 and audio.shape[1] >= 2:
-        audio = torch.mean(audio, dim=0).unsqueeze(0)
-    # audio = audio[0]
-    audio = audio.cpu().numpy()[0]
-    result = []
-    for k, v in chunks.items():
-        tag = v["split_time"].split(",")
-        if tag[0] != tag[1]:
-            result.append((v["slice"], audio[int(tag[0]):int(tag[1])]))
-    return result, sr

src/tts_vits/inference_main.py

@@ -1,55 +0,0 @@
-import io
-import logging
-import time
-from pathlib import Path
-
-import librosa
-import numpy as np
-import soundfile
-
-from inference import infer_tool
-from inference import slicer
-from inference.infer_tool import Svc
-import uuid
-
-logging.getLogger('numba').setLevel(logging.WARNING)
-# chunks_dict = infer_tool.read_temp("inference/chunks_temp.json")
-infer_tool.mkdir(["./results"])
-model_path = "vits_models/Haruhi_54000.pth"
-config_path = "configs/config.json"
-svc_model = Svc(model_path, config_path)
-
-
-def set_model_path(path):
-    global model_path
-    model_path = path
-
-
-def infer_to(spk, tran, voice):
-    slice_db = -40
-    
-    wav_format = 'wav'
-    # audio_file = io.BytesIO(voice)
-    audio_file = voice
-    chunks = slicer.cut(audio_file, db_thresh=slice_db)
-    # audio_file = io.BytesIO(voice)
-    audio_data, audio_sr = slicer.chunks2audio(audio_file, chunks)
-    audio = []
-    for (slice_tag, data) in audio_data:
-        print(f'#=====segment start, {round(len(data) / audio_sr, 3)}s======')
-        length = int(np.ceil(len(data) / audio_sr * svc_model.target_sample))
-        raw_path = io.BytesIO()
-        soundfile.write(raw_path, data, audio_sr, format="wav")
-        raw_path.seek(0)
-        if slice_tag:
-            print('jump empty segment')
-            _audio = np.zeros(length)
-        else:
-            out_audio, out_sr = svc_model.infer(spk, tran, raw_path)
-            _audio = out_audio.cpu().numpy()
-        audio.extend(list(_audio))
-    infer_tool.mkdir(["./vits_results"])
-    res_path = f'./vits_results/{tran}key_{spk}_{str(uuid.uuid4())}.{wav_format}'
-    soundfile.write(res_path, audio, svc_model.target_sample, format=wav_format)
-
-    return res_path

src/tts_vits/models.py

@@ -1,351 +0,0 @@
-import copy
-import math
-import torch
-from torch import nn
-from torch.nn import functional as F
-
-import attentions
-import commons
-import modules
-
-from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d
-from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
-from commons import init_weights, get_padding
-from vdecoder.hifigan.models import Generator
-from utils import f0_to_coarse
-
-class ResidualCouplingBlock(nn.Module):
-  def __init__(self,
-      channels,
-      hidden_channels,
-      kernel_size,
-      dilation_rate,
-      n_layers,
-      n_flows=4,
-      gin_channels=0):
-    super().__init__()
-    self.channels = channels
-    self.hidden_channels = hidden_channels
-    self.kernel_size = kernel_size
-    self.dilation_rate = dilation_rate
-    self.n_layers = n_layers
-    self.n_flows = n_flows
-    self.gin_channels = gin_channels
-
-    self.flows = nn.ModuleList()
-    for i in range(n_flows):
-      self.flows.append(modules.ResidualCouplingLayer(channels, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels, mean_only=True))
-      self.flows.append(modules.Flip())
-
-  def forward(self, x, x_mask, g=None, reverse=False):
-    if not reverse:
-      for flow in self.flows:
-        x, _ = flow(x, x_mask, g=g, reverse=reverse)
-    else:
-      for flow in reversed(self.flows):
-        x = flow(x, x_mask, g=g, reverse=reverse)
-    return x
-
-
-class Encoder(nn.Module):
-  def __init__(self,
-      in_channels,
-      out_channels,
-      hidden_channels,
-      kernel_size,
-      dilation_rate,
-      n_layers,
-      gin_channels=0):
-    super().__init__()
-    self.in_channels = in_channels
-    self.out_channels = out_channels
-    self.hidden_channels = hidden_channels
-    self.kernel_size = kernel_size
-    self.dilation_rate = dilation_rate
-    self.n_layers = n_layers
-    self.gin_channels = gin_channels
-
-    self.pre = nn.Conv1d(in_channels, hidden_channels, 1)
-    self.enc = modules.WN(hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels)
-    self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
-
-  def forward(self, x, x_lengths, g=None):
-    # print(x.shape,x_lengths.shape)
-    x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype)
-    x = self.pre(x) * x_mask
-    x = self.enc(x, x_mask, g=g)
-    stats = self.proj(x) * x_mask
-    m, logs = torch.split(stats, self.out_channels, dim=1)
-    z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask
-    return z, m, logs, x_mask
-
-
-class TextEncoder(nn.Module):
-  def __init__(self,
-      in_channels,
-      out_channels,
-      hidden_channels,
-      kernel_size,
-      dilation_rate,
-      n_layers,
-      gin_channels=0,
-      filter_channels=None,
-      n_heads=None,
-      p_dropout=None):
-    super().__init__()
-    self.in_channels = in_channels
-    self.out_channels = out_channels
-    self.hidden_channels = hidden_channels
-    self.kernel_size = kernel_size
-    self.dilation_rate = dilation_rate
-    self.n_layers = n_layers
-    self.gin_channels = gin_channels
-    self.pre = nn.Conv1d(in_channels, hidden_channels, 1)
-    self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
-    self.f0_emb = nn.Embedding(256, hidden_channels)
-
-    self.enc_ =  attentions.Encoder(
-        hidden_channels,
-        filter_channels,
-        n_heads,
-        n_layers,
-        kernel_size,
-        p_dropout)
-
-  def forward(self, x, x_lengths, f0=None):
-    x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype)
-    x = self.pre(x) * x_mask
-    x = x + self.f0_emb(f0).transpose(1,2)
-    x = self.enc_(x * x_mask, x_mask)
-    stats = self.proj(x) * x_mask
-    m, logs = torch.split(stats, self.out_channels, dim=1)
-    z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask
-
-    return z, m, logs, x_mask
-
-
-
-class DiscriminatorP(torch.nn.Module):
-    def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
-        super(DiscriminatorP, self).__init__()
-        self.period = period
-        self.use_spectral_norm = use_spectral_norm
-        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
-        self.convs = nn.ModuleList([
-            norm_f(Conv2d(1, 32, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
-            norm_f(Conv2d(32, 128, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
-            norm_f(Conv2d(128, 512, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
-            norm_f(Conv2d(512, 1024, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
-            norm_f(Conv2d(1024, 1024, (kernel_size, 1), 1, padding=(get_padding(kernel_size, 1), 0))),
-        ])
-        self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
-
-    def forward(self, x):
-        fmap = []
-
-        # 1d to 2d
-        b, c, t = x.shape
-        if t % self.period != 0: # pad first
-            n_pad = self.period - (t % self.period)
-            x = F.pad(x, (0, n_pad), "reflect")
-            t = t + n_pad
-        x = x.view(b, c, t // self.period, self.period)
-
-        for l in self.convs:
-            x = l(x)
-            x = F.leaky_relu(x, modules.LRELU_SLOPE)
-            fmap.append(x)
-        x = self.conv_post(x)
-        fmap.append(x)
-        x = torch.flatten(x, 1, -1)
-
-        return x, fmap
-
-
-class DiscriminatorS(torch.nn.Module):
-    def __init__(self, use_spectral_norm=False):
-        super(DiscriminatorS, self).__init__()
-        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
-        self.convs = nn.ModuleList([
-            norm_f(Conv1d(1, 16, 15, 1, padding=7)),
-            norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)),
-            norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)),
-            norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)),
-            norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)),
-            norm_f(Conv1d(1024, 1024, 5, 1, padding=2)),
-        ])
-        self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1))
-
-    def forward(self, x):
-        fmap = []
-
-        for l in self.convs:
-            x = l(x)
-            x = F.leaky_relu(x, modules.LRELU_SLOPE)
-            fmap.append(x)
-        x = self.conv_post(x)
-        fmap.append(x)
-        x = torch.flatten(x, 1, -1)
-
-        return x, fmap
-
-
-class MultiPeriodDiscriminator(torch.nn.Module):
-    def __init__(self, use_spectral_norm=False):
-        super(MultiPeriodDiscriminator, self).__init__()
-        periods = [2,3,5,7,11]
-
-        discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)]
-        discs = discs + [DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods]
-        self.discriminators = nn.ModuleList(discs)
-
-    def forward(self, y, y_hat):
-        y_d_rs = []
-        y_d_gs = []
-        fmap_rs = []
-        fmap_gs = []
-        for i, d in enumerate(self.discriminators):
-            y_d_r, fmap_r = d(y)
-            y_d_g, fmap_g = d(y_hat)
-            y_d_rs.append(y_d_r)
-            y_d_gs.append(y_d_g)
-            fmap_rs.append(fmap_r)
-            fmap_gs.append(fmap_g)
-
-        return y_d_rs, y_d_gs, fmap_rs, fmap_gs
-        
-        
-class SpeakerEncoder(torch.nn.Module):
-    def __init__(self, mel_n_channels=80, model_num_layers=3, model_hidden_size=256, model_embedding_size=256):
-        super(SpeakerEncoder, self).__init__()
-        self.lstm = nn.LSTM(mel_n_channels, model_hidden_size, model_num_layers, batch_first=True)
-        self.linear = nn.Linear(model_hidden_size, model_embedding_size)
-        self.relu = nn.ReLU()
-
-    def forward(self, mels):
-        self.lstm.flatten_parameters()
-        _, (hidden, _) = self.lstm(mels)
-        embeds_raw = self.relu(self.linear(hidden[-1]))
-        return embeds_raw / torch.norm(embeds_raw, dim=1, keepdim=True)
-        
-    def compute_partial_slices(self, total_frames, partial_frames, partial_hop):
-        mel_slices = []
-        for i in range(0, total_frames-partial_frames, partial_hop):
-            mel_range = torch.arange(i, i+partial_frames)
-            mel_slices.append(mel_range)
-            
-        return mel_slices
-    
-    def embed_utterance(self, mel, partial_frames=128, partial_hop=64):
-        mel_len = mel.size(1)
-        last_mel = mel[:,-partial_frames:]
-        
-        if mel_len > partial_frames:
-            mel_slices = self.compute_partial_slices(mel_len, partial_frames, partial_hop)
-            mels = list(mel[:,s] for s in mel_slices)
-            mels.append(last_mel)
-            mels = torch.stack(tuple(mels), 0).squeeze(1)
-        
-            with torch.no_grad():
-                partial_embeds = self(mels)
-            embed = torch.mean(partial_embeds, axis=0).unsqueeze(0)
-            #embed = embed / torch.linalg.norm(embed, 2)
-        else:
-            with torch.no_grad():
-                embed = self(last_mel)
-        
-        return embed
-
-
-class SynthesizerTrn(nn.Module):
-  """
-  Synthesizer for Training
-  """
-
-  def __init__(self, 
-    spec_channels,
-    segment_size,
-    inter_channels,
-    hidden_channels,
-    filter_channels,
-    n_heads,
-    n_layers,
-    kernel_size,
-    p_dropout,
-    resblock, 
-    resblock_kernel_sizes, 
-    resblock_dilation_sizes, 
-    upsample_rates, 
-    upsample_initial_channel, 
-    upsample_kernel_sizes,
-    gin_channels,
-    ssl_dim,
-    n_speakers,
-    **kwargs):
-
-    super().__init__()
-    self.spec_channels = spec_channels
-    self.inter_channels = inter_channels
-    self.hidden_channels = hidden_channels
-    self.filter_channels = filter_channels
-    self.n_heads = n_heads
-    self.n_layers = n_layers
-    self.kernel_size = kernel_size
-    self.p_dropout = p_dropout
-    self.resblock = resblock
-    self.resblock_kernel_sizes = resblock_kernel_sizes
-    self.resblock_dilation_sizes = resblock_dilation_sizes
-    self.upsample_rates = upsample_rates
-    self.upsample_initial_channel = upsample_initial_channel
-    self.upsample_kernel_sizes = upsample_kernel_sizes
-    self.segment_size = segment_size
-    self.gin_channels = gin_channels
-    self.ssl_dim = ssl_dim
-    self.emb_g = nn.Embedding(n_speakers, gin_channels)
-
-    self.enc_p_ = TextEncoder(ssl_dim, inter_channels, hidden_channels, 5, 1, 16,0, filter_channels, n_heads, p_dropout)
-    hps = {
-        "sampling_rate": 32000,
-        "inter_channels": 192,
-        "resblock": "1",
-        "resblock_kernel_sizes": [3, 7, 11],
-        "resblock_dilation_sizes": [[1, 3, 5], [1, 3, 5], [1, 3, 5]],
-        "upsample_rates": [10, 8, 2, 2],
-        "upsample_initial_channel": 512,
-        "upsample_kernel_sizes": [16, 16, 4, 4],
-        "gin_channels": 256,
-    }
-    self.dec = Generator(h=hps)
-    self.enc_q = Encoder(spec_channels, inter_channels, hidden_channels, 5, 1, 16, gin_channels=gin_channels)
-    self.flow = ResidualCouplingBlock(inter_channels, hidden_channels, 5, 1, 4, gin_channels=gin_channels)
-
-  def forward(self, c, f0, spec, g=None, mel=None, c_lengths=None, spec_lengths=None):
-    if c_lengths == None:
-      c_lengths = (torch.ones(c.size(0)) * c.size(-1)).to(c.device)
-    if spec_lengths == None:
-      spec_lengths = (torch.ones(spec.size(0)) * spec.size(-1)).to(spec.device)
-
-    g = self.emb_g(g).transpose(1,2)
-
-    z_ptemp, m_p, logs_p, _ = self.enc_p_(c, c_lengths, f0=f0_to_coarse(f0))
-    z, m_q, logs_q, spec_mask = self.enc_q(spec, spec_lengths, g=g) 
-
-    z_p = self.flow(z, spec_mask, g=g)
-    z_slice, pitch_slice, ids_slice = commons.rand_slice_segments_with_pitch(z, f0, spec_lengths, self.segment_size)
-
-    # o = self.dec(z_slice, g=g)
-    o = self.dec(z_slice, g=g, f0=pitch_slice)
-
-    return o, ids_slice, spec_mask, (z, z_p, m_p, logs_p, m_q, logs_q)
-
-  def infer(self, c, f0, g=None, mel=None, c_lengths=None):
-    if c_lengths == None:
-      c_lengths = (torch.ones(c.size(0)) * c.size(-1)).to(c.device)
-    g = self.emb_g(g).transpose(1,2)
-
-    z_p, m_p, logs_p, c_mask = self.enc_p_(c, c_lengths, f0=f0_to_coarse(f0))
-    z = self.flow(z_p, c_mask, g=g, reverse=True)
-
-    o = self.dec(z * c_mask, g=g, f0=f0)
-
-    return o

src/tts_vits/modules.py

@@ -1,342 +0,0 @@
-import copy
-import math
-import numpy as np
-import scipy
-import torch
-from torch import nn
-from torch.nn import functional as F
-
-from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d
-from torch.nn.utils import weight_norm, remove_weight_norm
-
-import commons
-from commons import init_weights, get_padding
-
-
-LRELU_SLOPE = 0.1
-
-
-class LayerNorm(nn.Module):
-  def __init__(self, channels, eps=1e-5):
-    super().__init__()
-    self.channels = channels
-    self.eps = eps
-
-    self.gamma = nn.Parameter(torch.ones(channels))
-    self.beta = nn.Parameter(torch.zeros(channels))
-
-  def forward(self, x):
-    x = x.transpose(1, -1)
-    x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
-    return x.transpose(1, -1)
-
- 
-class ConvReluNorm(nn.Module):
-  def __init__(self, in_channels, hidden_channels, out_channels, kernel_size, n_layers, p_dropout):
-    super().__init__()
-    self.in_channels = in_channels
-    self.hidden_channels = hidden_channels
-    self.out_channels = out_channels
-    self.kernel_size = kernel_size
-    self.n_layers = n_layers
-    self.p_dropout = p_dropout
-    assert n_layers > 1, "Number of layers should be larger than 0."
-
-    self.conv_layers = nn.ModuleList()
-    self.norm_layers = nn.ModuleList()
-    self.conv_layers.append(nn.Conv1d(in_channels, hidden_channels, kernel_size, padding=kernel_size//2))
-    self.norm_layers.append(LayerNorm(hidden_channels))
-    self.relu_drop = nn.Sequential(
-        nn.ReLU(),
-        nn.Dropout(p_dropout))
-    for _ in range(n_layers-1):
-      self.conv_layers.append(nn.Conv1d(hidden_channels, hidden_channels, kernel_size, padding=kernel_size//2))
-      self.norm_layers.append(LayerNorm(hidden_channels))
-    self.proj = nn.Conv1d(hidden_channels, out_channels, 1)
-    self.proj.weight.data.zero_()
-    self.proj.bias.data.zero_()
-
-  def forward(self, x, x_mask):
-    x_org = x
-    for i in range(self.n_layers):
-      x = self.conv_layers[i](x * x_mask)
-      x = self.norm_layers[i](x)
-      x = self.relu_drop(x)
-    x = x_org + self.proj(x)
-    return x * x_mask
-
-
-class DDSConv(nn.Module):
-  """
-  Dialted and Depth-Separable Convolution
-  """
-  def __init__(self, channels, kernel_size, n_layers, p_dropout=0.):
-    super().__init__()
-    self.channels = channels
-    self.kernel_size = kernel_size
-    self.n_layers = n_layers
-    self.p_dropout = p_dropout
-
-    self.drop = nn.Dropout(p_dropout)
-    self.convs_sep = nn.ModuleList()
-    self.convs_1x1 = nn.ModuleList()
-    self.norms_1 = nn.ModuleList()
-    self.norms_2 = nn.ModuleList()
-    for i in range(n_layers):
-      dilation = kernel_size ** i
-      padding = (kernel_size * dilation - dilation) // 2
-      self.convs_sep.append(nn.Conv1d(channels, channels, kernel_size, 
-          groups=channels, dilation=dilation, padding=padding
-      ))
-      self.convs_1x1.append(nn.Conv1d(channels, channels, 1))
-      self.norms_1.append(LayerNorm(channels))
-      self.norms_2.append(LayerNorm(channels))
-
-  def forward(self, x, x_mask, g=None):
-    if g is not None:
-      x = x + g
-    for i in range(self.n_layers):
-      y = self.convs_sep[i](x * x_mask)
-      y = self.norms_1[i](y)
-      y = F.gelu(y)
-      y = self.convs_1x1[i](y)
-      y = self.norms_2[i](y)
-      y = F.gelu(y)
-      y = self.drop(y)
-      x = x + y
-    return x * x_mask
-
-
-class WN(torch.nn.Module):
-  def __init__(self, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=0, p_dropout=0):
-    super(WN, self).__init__()
-    assert(kernel_size % 2 == 1)
-    self.hidden_channels =hidden_channels
-    self.kernel_size = kernel_size,
-    self.dilation_rate = dilation_rate
-    self.n_layers = n_layers
-    self.gin_channels = gin_channels
-    self.p_dropout = p_dropout
-
-    self.in_layers = torch.nn.ModuleList()
-    self.res_skip_layers = torch.nn.ModuleList()
-    self.drop = nn.Dropout(p_dropout)
-
-    if gin_channels != 0:
-      cond_layer = torch.nn.Conv1d(gin_channels, 2*hidden_channels*n_layers, 1)
-      self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight')
-
-    for i in range(n_layers):
-      dilation = dilation_rate ** i
-      padding = int((kernel_size * dilation - dilation) / 2)
-      in_layer = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, kernel_size,
-                                 dilation=dilation, padding=padding)
-      in_layer = torch.nn.utils.weight_norm(in_layer, name='weight')
-      self.in_layers.append(in_layer)
-
-      # last one is not necessary
-      if i < n_layers - 1:
-        res_skip_channels = 2 * hidden_channels
-      else:
-        res_skip_channels = hidden_channels
-
-      res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1)
-      res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight')
-      self.res_skip_layers.append(res_skip_layer)
-
-  def forward(self, x, x_mask, g=None, **kwargs):
-    output = torch.zeros_like(x)
-    n_channels_tensor = torch.IntTensor([self.hidden_channels])
-
-    if g is not None:
-      g = self.cond_layer(g)
-
-    for i in range(self.n_layers):
-      x_in = self.in_layers[i](x)
-      if g is not None:
-        cond_offset = i * 2 * self.hidden_channels
-        g_l = g[:,cond_offset:cond_offset+2*self.hidden_channels,:]
-      else:
-        g_l = torch.zeros_like(x_in)
-
-      acts = commons.fused_add_tanh_sigmoid_multiply(
-          x_in,
-          g_l,
-          n_channels_tensor)
-      acts = self.drop(acts)
-
-      res_skip_acts = self.res_skip_layers[i](acts)
-      if i < self.n_layers - 1:
-        res_acts = res_skip_acts[:,:self.hidden_channels,:]
-        x = (x + res_acts) * x_mask
-        output = output + res_skip_acts[:,self.hidden_channels:,:]
-      else:
-        output = output + res_skip_acts
-    return output * x_mask
-
-  def remove_weight_norm(self):
-    if self.gin_channels != 0:
-      torch.nn.utils.remove_weight_norm(self.cond_layer)
-    for l in self.in_layers:
-      torch.nn.utils.remove_weight_norm(l)
-    for l in self.res_skip_layers:
-     torch.nn.utils.remove_weight_norm(l)
-
-
-class ResBlock1(torch.nn.Module):
-    def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)):
-        super(ResBlock1, self).__init__()
-        self.convs1 = nn.ModuleList([
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
-                               padding=get_padding(kernel_size, dilation[0]))),
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
-                               padding=get_padding(kernel_size, dilation[1]))),
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2],
-                               padding=get_padding(kernel_size, dilation[2])))
-        ])
-        self.convs1.apply(init_weights)
-
-        self.convs2 = nn.ModuleList([
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
-                               padding=get_padding(kernel_size, 1))),
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
-                               padding=get_padding(kernel_size, 1))),
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
-                               padding=get_padding(kernel_size, 1)))
-        ])
-        self.convs2.apply(init_weights)
-
-    def forward(self, x, x_mask=None):
-        for c1, c2 in zip(self.convs1, self.convs2):
-            xt = F.leaky_relu(x, LRELU_SLOPE)
-            if x_mask is not None:
-                xt = xt * x_mask
-            xt = c1(xt)
-            xt = F.leaky_relu(xt, LRELU_SLOPE)
-            if x_mask is not None:
-                xt = xt * x_mask
-            xt = c2(xt)
-            x = xt + x
-        if x_mask is not None:
-            x = x * x_mask
-        return x
-
-    def remove_weight_norm(self):
-        for l in self.convs1:
-            remove_weight_norm(l)
-        for l in self.convs2:
-            remove_weight_norm(l)
-
-
-class ResBlock2(torch.nn.Module):
-    def __init__(self, channels, kernel_size=3, dilation=(1, 3)):
-        super(ResBlock2, self).__init__()
-        self.convs = nn.ModuleList([
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
-                               padding=get_padding(kernel_size, dilation[0]))),
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
-                               padding=get_padding(kernel_size, dilation[1])))
-        ])
-        self.convs.apply(init_weights)
-
-    def forward(self, x, x_mask=None):
-        for c in self.convs:
-            xt = F.leaky_relu(x, LRELU_SLOPE)
-            if x_mask is not None:
-                xt = xt * x_mask
-            xt = c(xt)
-            x = xt + x
-        if x_mask is not None:
-            x = x * x_mask
-        return x
-
-    def remove_weight_norm(self):
-        for l in self.convs:
-            remove_weight_norm(l)
-
-
-class Log(nn.Module):
-  def forward(self, x, x_mask, reverse=False, **kwargs):
-    if not reverse:
-      y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask
-      logdet = torch.sum(-y, [1, 2])
-      return y, logdet
-    else:
-      x = torch.exp(x) * x_mask
-      return x
-    
-
-class Flip(nn.Module):
-  def forward(self, x, *args, reverse=False, **kwargs):
-    x = torch.flip(x, [1])
-    if not reverse:
-      logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device)
-      return x, logdet
-    else:
-      return x
-
-
-class ElementwiseAffine(nn.Module):
-  def __init__(self, channels):
-    super().__init__()
-    self.channels = channels
-    self.m = nn.Parameter(torch.zeros(channels,1))
-    self.logs = nn.Parameter(torch.zeros(channels,1))
-
-  def forward(self, x, x_mask, reverse=False, **kwargs):
-    if not reverse:
-      y = self.m + torch.exp(self.logs) * x
-      y = y * x_mask
-      logdet = torch.sum(self.logs * x_mask, [1,2])
-      return y, logdet
-    else:
-      x = (x - self.m) * torch.exp(-self.logs) * x_mask
-      return x
-
-
-class ResidualCouplingLayer(nn.Module):
-  def __init__(self,
-      channels,
-      hidden_channels,
-      kernel_size,
-      dilation_rate,
-      n_layers,
-      p_dropout=0,
-      gin_channels=0,
-      mean_only=False):
-    assert channels % 2 == 0, "channels should be divisible by 2"
-    super().__init__()
-    self.channels = channels
-    self.hidden_channels = hidden_channels
-    self.kernel_size = kernel_size
-    self.dilation_rate = dilation_rate
-    self.n_layers = n_layers
-    self.half_channels = channels // 2
-    self.mean_only = mean_only
-
-    self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1)
-    self.enc = WN(hidden_channels, kernel_size, dilation_rate, n_layers, p_dropout=p_dropout, gin_channels=gin_channels)
-    self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1)
-    self.post.weight.data.zero_()
-    self.post.bias.data.zero_()
-
-  def forward(self, x, x_mask, g=None, reverse=False):
-    x0, x1 = torch.split(x, [self.half_channels]*2, 1)
-    h = self.pre(x0) * x_mask
-    h = self.enc(h, x_mask, g=g)
-    stats = self.post(h) * x_mask
-    if not self.mean_only:
-      m, logs = torch.split(stats, [self.half_channels]*2, 1)
-    else:
-      m = stats
-      logs = torch.zeros_like(m)
-
-    if not reverse:
-      x1 = m + x1 * torch.exp(logs) * x_mask
-      x = torch.cat([x0, x1], 1)
-      logdet = torch.sum(logs, [1,2])
-      return x, logdet
-    else:
-      x1 = (x1 - m) * torch.exp(-logs) * x_mask
-      x = torch.cat([x0, x1], 1)
-      return x

src/tts_vits/requirements.txt

@@ -1,16 +0,0 @@
-Flask==2.1.2
-Flask_Cors==3.0.10
-gradio==3.4.1
-playsound==1.3.0
-PyAudio==0.2.12
-pydub==0.25.1
-pyworld==0.3.3
-requests==2.28.1
-scipy==1.7.3
-sounddevice==0.4.5
-SoundFile==0.10.3.post1
-starlette==0.19.1
-torchaudio==0.10.0
-tqdm==4.63.0
-scikit-maad
-praat-parselmouth

src/tts_vits/utils.py

@@ -1,338 +0,0 @@
-import os
-import glob
-import sys
-import argparse
-import logging
-import json
-import subprocess
-
-import librosa
-import numpy as np
-import torchaudio
-from scipy.io.wavfile import read
-import torch
-import torchvision
-from torch.nn import functional as F
-from commons import sequence_mask
-from hubert import hubert_model
-MATPLOTLIB_FLAG = False
-
-logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
-logger = logging
-
-f0_bin = 256
-f0_max = 1100.0
-f0_min = 50.0
-f0_mel_min = 1127 * np.log(1 + f0_min / 700)
-f0_mel_max = 1127 * np.log(1 + f0_max / 700)
-
-def f0_to_coarse(f0):
-  is_torch = isinstance(f0, torch.Tensor)
-  f0_mel = 1127 * (1 + f0 / 700).log() if is_torch else 1127 * np.log(1 + f0 / 700)
-  f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * (f0_bin - 2) / (f0_mel_max - f0_mel_min) + 1
-
-  f0_mel[f0_mel <= 1] = 1
-  f0_mel[f0_mel > f0_bin - 1] = f0_bin - 1
-  f0_coarse = (f0_mel + 0.5).long() if is_torch else np.rint(f0_mel).astype(np.int)
-  assert f0_coarse.max() <= 255 and f0_coarse.min() >= 1, (f0_coarse.max(), f0_coarse.min())
-  return f0_coarse
-
-
-def get_hubert_model(rank=None):
-
-  hubert_soft = hubert_model.hubert_soft("hubert/hubert-soft-0d54a1f4.pt")
-  if rank is not None:
-    hubert_soft = hubert_soft.cuda(rank)
-  return hubert_soft
-
-def get_hubert_content(hmodel, y=None, path=None):
-  if path is not None:
-    source, sr = torchaudio.load(path)
-    source = torchaudio.functional.resample(source, sr, 16000)
-    if len(source.shape) == 2 and source.shape[1] >= 2:
-      source = torch.mean(source, dim=0).unsqueeze(0)
-  else:
-    source = y
-  source = source.unsqueeze(0)
-  with torch.inference_mode():
-    units = hmodel.units(source)
-    return units.transpose(1,2)
-
-
-def get_content(cmodel, y):
-    with torch.no_grad():
-        c = cmodel.extract_features(y.squeeze(1))[0]
-    c = c.transpose(1, 2)
-    return c
-
-
-
-def transform(mel, height): # 68-92
-    #r = np.random.random()
-    #rate = r * 0.3 + 0.85 # 0.85-1.15
-    #height = int(mel.size(-2) * rate)
-    tgt = torchvision.transforms.functional.resize(mel, (height, mel.size(-1)))
-    if height >= mel.size(-2):
-        return tgt[:, :mel.size(-2), :]
-    else:
-        silence = tgt[:,-1:,:].repeat(1,mel.size(-2)-height,1)
-        silence += torch.randn_like(silence) / 10
-        return torch.cat((tgt, silence), 1)
-
-
-def stretch(mel, width): # 0.5-2
-    return torchvision.transforms.functional.resize(mel, (mel.size(-2), width))
-
-
-def load_checkpoint(checkpoint_path, model, optimizer=None):
-  assert os.path.isfile(checkpoint_path)
-  checkpoint_dict = torch.load(checkpoint_path, map_location='cpu')
-  iteration = checkpoint_dict['iteration']
-  learning_rate = checkpoint_dict['learning_rate']
-  if iteration is None:
-    iteration = 1
-  if learning_rate is None:
-    learning_rate = 0.0002
-  if optimizer is not None and checkpoint_dict['optimizer'] is not None:
-    optimizer.load_state_dict(checkpoint_dict['optimizer'])
-  saved_state_dict = checkpoint_dict['model']
-  if hasattr(model, 'module'):
-    state_dict = model.module.state_dict()
-  else:
-    state_dict = model.state_dict()
-  new_state_dict= {}
-  for k, v in state_dict.items():
-    try:
-      new_state_dict[k] = saved_state_dict[k]
-    except:
-      logger.info("%s is not in the checkpoint" % k)
-      new_state_dict[k] = v
-  if hasattr(model, 'module'):
-    model.module.load_state_dict(new_state_dict)
-  else:
-    model.load_state_dict(new_state_dict)
-  logger.info("Loaded checkpoint '{}' (iteration {})" .format(
-    checkpoint_path, iteration))
-  return model, optimizer, learning_rate, iteration
-
-
-def save_checkpoint(model, optimizer, learning_rate, iteration, checkpoint_path):
-  # ckptname = checkpoint_path.split(os.sep)[-1]
-  # newest_step = int(ckptname.split(".")[0].split("_")[1])
-  # val_steps = 2000
-  # last_ckptname = checkpoint_path.replace(str(newest_step), str(newest_step - val_steps*3))
-  # if newest_step >= val_steps*3:
-  #   os.system(f"rm {last_ckptname}")
-  logger.info("Saving model and optimizer state at iteration {} to {}".format(
-    iteration, checkpoint_path))
-  if hasattr(model, 'module'):
-    state_dict = model.module.state_dict()
-  else:
-    state_dict = model.state_dict()
-  torch.save({'model': state_dict,
-              'iteration': iteration,
-              'optimizer': optimizer.state_dict(),
-              'learning_rate': learning_rate}, checkpoint_path)
-
-
-def summarize(writer, global_step, scalars={}, histograms={}, images={}, audios={}, audio_sampling_rate=22050):
-  for k, v in scalars.items():
-    writer.add_scalar(k, v, global_step)
-  for k, v in histograms.items():
-    writer.add_histogram(k, v, global_step)
-  for k, v in images.items():
-    writer.add_image(k, v, global_step, dataformats='HWC')
-  for k, v in audios.items():
-    writer.add_audio(k, v, global_step, audio_sampling_rate)
-
-
-def latest_checkpoint_path(dir_path, regex="G_*.pth"):
-  f_list = glob.glob(os.path.join(dir_path, regex))
-  f_list.sort(key=lambda f: int("".join(filter(str.isdigit, f))))
-  x = f_list[-1]
-  print(x)
-  return x
-
-
-def plot_spectrogram_to_numpy(spectrogram):
-  global MATPLOTLIB_FLAG
-  if not MATPLOTLIB_FLAG:
-    import matplotlib
-    matplotlib.use("Agg")
-    MATPLOTLIB_FLAG = True
-    mpl_logger = logging.getLogger('matplotlib')
-    mpl_logger.setLevel(logging.WARNING)
-  import matplotlib.pylab as plt
-  import numpy as np
-
-  fig, ax = plt.subplots(figsize=(10,2))
-  im = ax.imshow(spectrogram, aspect="auto", origin="lower",
-                  interpolation='none')
-  plt.colorbar(im, ax=ax)
-  plt.xlabel("Frames")
-  plt.ylabel("Channels")
-  plt.tight_layout()
-
-  fig.canvas.draw()
-  data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
-  data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
-  plt.close()
-  return data
-
-
-def plot_alignment_to_numpy(alignment, info=None):
-  global MATPLOTLIB_FLAG
-  if not MATPLOTLIB_FLAG:
-    import matplotlib
-    matplotlib.use("Agg")
-    MATPLOTLIB_FLAG = True
-    mpl_logger = logging.getLogger('matplotlib')
-    mpl_logger.setLevel(logging.WARNING)
-  import matplotlib.pylab as plt
-  import numpy as np
-
-  fig, ax = plt.subplots(figsize=(6, 4))
-  im = ax.imshow(alignment.transpose(), aspect='auto', origin='lower',
-                  interpolation='none')
-  fig.colorbar(im, ax=ax)
-  xlabel = 'Decoder timestep'
-  if info is not None:
-      xlabel += '\n\n' + info
-  plt.xlabel(xlabel)
-  plt.ylabel('Encoder timestep')
-  plt.tight_layout()
-
-  fig.canvas.draw()
-  data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
-  data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
-  plt.close()
-  return data
-
-
-def load_wav_to_torch(full_path):
-  sampling_rate, data = read(full_path)
-  return torch.FloatTensor(data.astype(np.float32)), sampling_rate
-
-
-def load_filepaths_and_text(filename, split="|"):
-  with open(filename, encoding='utf-8') as f:
-    filepaths_and_text = [line.strip().split(split) for line in f]
-  return filepaths_and_text
-
-
-def get_hparams(init=True):
-  parser = argparse.ArgumentParser()
-  parser.add_argument('-c', '--config', type=str, default="./configs/base.json",
-                      help='JSON file for configuration')
-  parser.add_argument('-m', '--model', type=str, required=True,
-                      help='Model name')
-
-  args = parser.parse_args()
-  model_dir = os.path.join("./logs", args.model)
-
-  if not os.path.exists(model_dir):
-    os.makedirs(model_dir)
-
-  config_path = args.config
-  config_save_path = os.path.join(model_dir, "config.json")
-  if init:
-    with open(config_path, "r") as f:
-      data = f.read()
-    with open(config_save_path, "w") as f:
-      f.write(data)
-  else:
-    with open(config_save_path, "r") as f:
-      data = f.read()
-  config = json.loads(data)
-
-  hparams = HParams(**config)
-  hparams.model_dir = model_dir
-  return hparams
-
-
-def get_hparams_from_dir(model_dir):
-  config_save_path = os.path.join(model_dir, "config.json")
-  with open(config_save_path, "r") as f:
-    data = f.read()
-  config = json.loads(data)
-
-  hparams =HParams(**config)
-  hparams.model_dir = model_dir
-  return hparams
-
-
-def get_hparams_from_file(config_path):
-  with open(config_path, "r") as f:
-    data = f.read()
-  config = json.loads(data)
-
-  hparams =HParams(**config)
-  return hparams
-
-
-def check_git_hash(model_dir):
-  source_dir = os.path.dirname(os.path.realpath(__file__))
-  if not os.path.exists(os.path.join(source_dir, ".git")):
-    logger.warn("{} is not a git repository, therefore hash value comparison will be ignored.".format(
-      source_dir
-    ))
-    return
-
-  cur_hash = subprocess.getoutput("git rev-parse HEAD")
-
-  path = os.path.join(model_dir, "githash")
-  if os.path.exists(path):
-    saved_hash = open(path).read()
-    if saved_hash != cur_hash:
-      logger.warn("git hash values are different. {}(saved) != {}(current)".format(
-        saved_hash[:8], cur_hash[:8]))
-  else:
-    open(path, "w").write(cur_hash)
-
-
-def get_logger(model_dir, filename="train.log"):
-  global logger
-  logger = logging.getLogger(os.path.basename(model_dir))
-  logger.setLevel(logging.DEBUG)
-
-  formatter = logging.Formatter("%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s")
-  if not os.path.exists(model_dir):
-    os.makedirs(model_dir)
-  h = logging.FileHandler(os.path.join(model_dir, filename))
-  h.setLevel(logging.DEBUG)
-  h.setFormatter(formatter)
-  logger.addHandler(h)
-  return logger
-
-
-class HParams():
-  def __init__(self, **kwargs):
-    for k, v in kwargs.items():
-      if type(v) == dict:
-        v = HParams(**v)
-      self[k] = v
-
-  def keys(self):
-    return self.__dict__.keys()
-
-  def items(self):
-    return self.__dict__.items()
-
-  def values(self):
-    return self.__dict__.values()
-
-  def __len__(self):
-    return len(self.__dict__)
-
-  def __getitem__(self, key):
-    return getattr(self, key)
-
-  def __setitem__(self, key, value):
-    return setattr(self, key, value)
-
-  def __contains__(self, key):
-    return key in self.__dict__
-
-  def __repr__(self):
-    return self.__dict__.__repr__()
-

src/tts_vits/vdecoder/hifigan/env.py

@@ -1,15 +0,0 @@
-import os
-import shutil
-
-
-class AttrDict(dict):
-    def __init__(self, *args, **kwargs):
-        super(AttrDict, self).__init__(*args, **kwargs)
-        self.__dict__ = self
-
-
-def build_env(config, config_name, path):
-    t_path = os.path.join(path, config_name)
-    if config != t_path:
-        os.makedirs(path, exist_ok=True)
-        shutil.copyfile(config, os.path.join(path, config_name))

src/tts_vits/vdecoder/hifigan/models.py

@@ -1,503 +0,0 @@
-import os
-import json
-from .env import AttrDict
-import numpy as np
-import torch
-import torch.nn.functional as F
-import torch.nn as nn
-from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d
-from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
-from .utils import init_weights, get_padding
-
-LRELU_SLOPE = 0.1
-
-
-def load_model(model_path, device='cuda'):
-    config_file = os.path.join(os.path.split(model_path)[0], 'config.json')
-    with open(config_file) as f:
-        data = f.read()
-
-    global h
-    json_config = json.loads(data)
-    h = AttrDict(json_config)
-
-    generator = Generator(h).to(device)
-
-    cp_dict = torch.load(model_path)
-    generator.load_state_dict(cp_dict['generator'])
-    generator.eval()
-    generator.remove_weight_norm()
-    del cp_dict
-    return generator, h
-
-
-class ResBlock1(torch.nn.Module):
-    def __init__(self, h, channels, kernel_size=3, dilation=(1, 3, 5)):
-        super(ResBlock1, self).__init__()
-        self.h = h
-        self.convs1 = nn.ModuleList([
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
-                               padding=get_padding(kernel_size, dilation[0]))),
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
-                               padding=get_padding(kernel_size, dilation[1]))),
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2],
-                               padding=get_padding(kernel_size, dilation[2])))
-        ])
-        self.convs1.apply(init_weights)
-
-        self.convs2 = nn.ModuleList([
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
-                               padding=get_padding(kernel_size, 1))),
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
-                               padding=get_padding(kernel_size, 1))),
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
-                               padding=get_padding(kernel_size, 1)))
-        ])
-        self.convs2.apply(init_weights)
-
-    def forward(self, x):
-        for c1, c2 in zip(self.convs1, self.convs2):
-            xt = F.leaky_relu(x, LRELU_SLOPE)
-            xt = c1(xt)
-            xt = F.leaky_relu(xt, LRELU_SLOPE)
-            xt = c2(xt)
-            x = xt + x
-        return x
-
-    def remove_weight_norm(self):
-        for l in self.convs1:
-            remove_weight_norm(l)
-        for l in self.convs2:
-            remove_weight_norm(l)
-
-
-class ResBlock2(torch.nn.Module):
-    def __init__(self, h, channels, kernel_size=3, dilation=(1, 3)):
-        super(ResBlock2, self).__init__()
-        self.h = h
-        self.convs = nn.ModuleList([
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
-                               padding=get_padding(kernel_size, dilation[0]))),
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
-                               padding=get_padding(kernel_size, dilation[1])))
-        ])
-        self.convs.apply(init_weights)
-
-    def forward(self, x):
-        for c in self.convs:
-            xt = F.leaky_relu(x, LRELU_SLOPE)
-            xt = c(xt)
-            x = xt + x
-        return x
-
-    def remove_weight_norm(self):
-        for l in self.convs:
-            remove_weight_norm(l)
-
-
-def padDiff(x):
-    return F.pad(F.pad(x, (0,0,-1,1), 'constant', 0) - x, (0,0,0,-1), 'constant', 0)
-
-class SineGen(torch.nn.Module):
-    """ Definition of sine generator
-    SineGen(samp_rate, harmonic_num = 0,
-            sine_amp = 0.1, noise_std = 0.003,
-            voiced_threshold = 0,
-            flag_for_pulse=False)
-    samp_rate: sampling rate in Hz
-    harmonic_num: number of harmonic overtones (default 0)
-    sine_amp: amplitude of sine-wavefrom (default 0.1)
-    noise_std: std of Gaussian noise (default 0.003)
-    voiced_thoreshold: F0 threshold for U/V classification (default 0)
-    flag_for_pulse: this SinGen is used inside PulseGen (default False)
-    Note: when flag_for_pulse is True, the first time step of a voiced
-        segment is always sin(np.pi) or cos(0)
-    """
-
-    def __init__(self, samp_rate, harmonic_num=0,
-                 sine_amp=0.1, noise_std=0.003,
-                 voiced_threshold=0,
-                 flag_for_pulse=False):
-        super(SineGen, self).__init__()
-        self.sine_amp = sine_amp
-        self.noise_std = noise_std
-        self.harmonic_num = harmonic_num
-        self.dim = self.harmonic_num + 1
-        self.sampling_rate = samp_rate
-        self.voiced_threshold = voiced_threshold
-        self.flag_for_pulse = flag_for_pulse
-
-    def _f02uv(self, f0):
-        # generate uv signal
-        uv = (f0 > self.voiced_threshold).type(torch.float32)
-        return uv
-
-    def _f02sine(self, f0_values):
-        """ f0_values: (batchsize, length, dim)
-            where dim indicates fundamental tone and overtones
-        """
-        # convert to F0 in rad. The interger part n can be ignored
-        # because 2 * np.pi * n doesn't affect phase
-        rad_values = (f0_values / self.sampling_rate) % 1
-
-        # initial phase noise (no noise for fundamental component)
-        rand_ini = torch.rand(f0_values.shape[0], f0_values.shape[2], \
-                              device=f0_values.device)
-        rand_ini[:, 0] = 0
-        rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini
-
-        # instantanouse phase sine[t] = sin(2*pi \sum_i=1 ^{t} rad)
-        if not self.flag_for_pulse:
-            # for normal case
-
-            # To prevent torch.cumsum numerical overflow,
-            # it is necessary to add -1 whenever \sum_k=1^n rad_value_k > 1.
-            # Buffer tmp_over_one_idx indicates the time step to add -1.
-            # This will not change F0 of sine because (x-1) * 2*pi = x * 2*pi
-            tmp_over_one = torch.cumsum(rad_values, 1) % 1
-            tmp_over_one_idx = (padDiff(tmp_over_one)) < 0
-            cumsum_shift = torch.zeros_like(rad_values)
-            cumsum_shift[:, 1:, :] = tmp_over_one_idx * -1.0
-
-            sines = torch.sin(torch.cumsum(rad_values + cumsum_shift, dim=1)
-                              * 2 * np.pi)
-        else:
-            # If necessary, make sure that the first time step of every
-            # voiced segments is sin(pi) or cos(0)
-            # This is used for pulse-train generation
-
-            # identify the last time step in unvoiced segments
-            uv = self._f02uv(f0_values)
-            uv_1 = torch.roll(uv, shifts=-1, dims=1)
-            uv_1[:, -1, :] = 1
-            u_loc = (uv < 1) * (uv_1 > 0)
-
-            # get the instantanouse phase
-            tmp_cumsum = torch.cumsum(rad_values, dim=1)
-            # different batch needs to be processed differently
-            for idx in range(f0_values.shape[0]):
-                temp_sum = tmp_cumsum[idx, u_loc[idx, :, 0], :]
-                temp_sum[1:, :] = temp_sum[1:, :] - temp_sum[0:-1, :]
-                # stores the accumulation of i.phase within
-                # each voiced segments
-                tmp_cumsum[idx, :, :] = 0
-                tmp_cumsum[idx, u_loc[idx, :, 0], :] = temp_sum
-
-            # rad_values - tmp_cumsum: remove the accumulation of i.phase
-            # within the previous voiced segment.
-            i_phase = torch.cumsum(rad_values - tmp_cumsum, dim=1)
-
-            # get the sines
-            sines = torch.cos(i_phase * 2 * np.pi)
-        return sines
-
-    def forward(self, f0):
-        """ sine_tensor, uv = forward(f0)
-        input F0: tensor(batchsize=1, length, dim=1)
-                  f0 for unvoiced steps should be 0
-        output sine_tensor: tensor(batchsize=1, length, dim)
-        output uv: tensor(batchsize=1, length, 1)
-        """
-        with torch.no_grad():
-            f0_buf = torch.zeros(f0.shape[0], f0.shape[1], self.dim,
-                                 device=f0.device)
-            # fundamental component
-            fn = torch.multiply(f0, torch.FloatTensor([[range(1, self.harmonic_num + 2)]]).to(f0.device))
-
-            # generate sine waveforms
-            sine_waves = self._f02sine(fn) * self.sine_amp
-
-            # generate uv signal
-            # uv = torch.ones(f0.shape)
-            # uv = uv * (f0 > self.voiced_threshold)
-            uv = self._f02uv(f0)
-
-            # noise: for unvoiced should be similar to sine_amp
-            #        std = self.sine_amp/3 -> max value ~ self.sine_amp
-            # .       for voiced regions is self.noise_std
-            noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3
-            noise = noise_amp * torch.randn_like(sine_waves)
-
-            # first: set the unvoiced part to 0 by uv
-            # then: additive noise
-            sine_waves = sine_waves * uv + noise
-        return sine_waves, uv, noise
-
-
-class SourceModuleHnNSF(torch.nn.Module):
-    """ SourceModule for hn-nsf
-    SourceModule(sampling_rate, harmonic_num=0, sine_amp=0.1,
-                 add_noise_std=0.003, voiced_threshod=0)
-    sampling_rate: sampling_rate in Hz
-    harmonic_num: number of harmonic above F0 (default: 0)
-    sine_amp: amplitude of sine source signal (default: 0.1)
-    add_noise_std: std of additive Gaussian noise (default: 0.003)
-        note that amplitude of noise in unvoiced is decided
-        by sine_amp
-    voiced_threshold: threhold to set U/V given F0 (default: 0)
-    Sine_source, noise_source = SourceModuleHnNSF(F0_sampled)
-    F0_sampled (batchsize, length, 1)
-    Sine_source (batchsize, length, 1)
-    noise_source (batchsize, length 1)
-    uv (batchsize, length, 1)
-    """
-
-    def __init__(self, sampling_rate, harmonic_num=0, sine_amp=0.1,
-                 add_noise_std=0.003, voiced_threshod=0):
-        super(SourceModuleHnNSF, self).__init__()
-
-        self.sine_amp = sine_amp
-        self.noise_std = add_noise_std
-
-        # to produce sine waveforms
-        self.l_sin_gen = SineGen(sampling_rate, harmonic_num,
-                                 sine_amp, add_noise_std, voiced_threshod)
-
-        # to merge source harmonics into a single excitation
-        self.l_linear = torch.nn.Linear(harmonic_num + 1, 1)
-        self.l_tanh = torch.nn.Tanh()
-
-    def forward(self, x):
-        """
-        Sine_source, noise_source = SourceModuleHnNSF(F0_sampled)
-        F0_sampled (batchsize, length, 1)
-        Sine_source (batchsize, length, 1)
-        noise_source (batchsize, length 1)
-        """
-        # source for harmonic branch
-        sine_wavs, uv, _ = self.l_sin_gen(x)
-        sine_merge = self.l_tanh(self.l_linear(sine_wavs))
-
-        # source for noise branch, in the same shape as uv
-        noise = torch.randn_like(uv) * self.sine_amp / 3
-        return sine_merge, noise, uv
-
-
-class Generator(torch.nn.Module):
-    def __init__(self, h):
-        super(Generator, self).__init__()
-        self.h = h
-
-        self.num_kernels = len(h["resblock_kernel_sizes"])
-        self.num_upsamples = len(h["upsample_rates"])
-        self.f0_upsamp = torch.nn.Upsample(scale_factor=np.prod(h["upsample_rates"]))
-        self.m_source = SourceModuleHnNSF(
-            sampling_rate=h["sampling_rate"],
-            harmonic_num=8)
-        self.noise_convs = nn.ModuleList()
-        self.conv_pre = weight_norm(Conv1d(h["inter_channels"], h["upsample_initial_channel"], 7, 1, padding=3))
-        resblock = ResBlock1 if h["resblock"] == '1' else ResBlock2
-        self.ups = nn.ModuleList()
-        for i, (u, k) in enumerate(zip(h["upsample_rates"], h["upsample_kernel_sizes"])):
-            c_cur = h["upsample_initial_channel"] // (2 ** (i + 1))
-            self.ups.append(weight_norm(
-                ConvTranspose1d(h["upsample_initial_channel"] // (2 ** i), h["upsample_initial_channel"] // (2 ** (i + 1)),
-                                k, u, padding=(k - u) // 2)))
-            if i + 1 < len(h["upsample_rates"]):  #
-                stride_f0 = np.prod(h["upsample_rates"][i + 1:])
-                self.noise_convs.append(Conv1d(
-                    1, c_cur, kernel_size=stride_f0 * 2, stride=stride_f0, padding=stride_f0 // 2))
-            else:
-                self.noise_convs.append(Conv1d(1, c_cur, kernel_size=1))
-        self.resblocks = nn.ModuleList()
-        for i in range(len(self.ups)):
-            ch = h["upsample_initial_channel"] // (2 ** (i + 1))
-            for j, (k, d) in enumerate(zip(h["resblock_kernel_sizes"], h["resblock_dilation_sizes"])):
-                self.resblocks.append(resblock(h, ch, k, d))
-
-        self.conv_post = weight_norm(Conv1d(ch, 1, 7, 1, padding=3))
-        self.ups.apply(init_weights)
-        self.conv_post.apply(init_weights)
-        self.cond = nn.Conv1d(h['gin_channels'], h['upsample_initial_channel'], 1)
-
-    def forward(self, x, f0, g=None):
-        # print(1,x.shape,f0.shape,f0[:, None].shape)
-        f0 = self.f0_upsamp(f0[:, None]).transpose(1, 2)  # bs,n,t
-        # print(2,f0.shape)
-        har_source, noi_source, uv = self.m_source(f0)
-        har_source = har_source.transpose(1, 2)
-        x = self.conv_pre(x)
-        x = x + self.cond(g)
-        # print(124,x.shape,har_source.shape)
-        for i in range(self.num_upsamples):
-            x = F.leaky_relu(x, LRELU_SLOPE)
-            # print(3,x.shape)
-            x = self.ups[i](x)
-            x_source = self.noise_convs[i](har_source)
-            # print(4,x_source.shape,har_source.shape,x.shape)
-            x = x + x_source
-            xs = None
-            for j in range(self.num_kernels):
-                if xs is None:
-                    xs = self.resblocks[i * self.num_kernels + j](x)
-                else:
-                    xs += self.resblocks[i * self.num_kernels + j](x)
-            x = xs / self.num_kernels
-        x = F.leaky_relu(x)
-        x = self.conv_post(x)
-        x = torch.tanh(x)
-
-        return x
-
-    def remove_weight_norm(self):
-        print('Removing weight norm...')
-        for l in self.ups:
-            remove_weight_norm(l)
-        for l in self.resblocks:
-            l.remove_weight_norm()
-        remove_weight_norm(self.conv_pre)
-        remove_weight_norm(self.conv_post)
-
-
-class DiscriminatorP(torch.nn.Module):
-    def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
-        super(DiscriminatorP, self).__init__()
-        self.period = period
-        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
-        self.convs = nn.ModuleList([
-            norm_f(Conv2d(1, 32, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))),
-            norm_f(Conv2d(32, 128, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))),
-            norm_f(Conv2d(128, 512, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))),
-            norm_f(Conv2d(512, 1024, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))),
-            norm_f(Conv2d(1024, 1024, (kernel_size, 1), 1, padding=(2, 0))),
-        ])
-        self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
-
-    def forward(self, x):
-        fmap = []
-
-        # 1d to 2d
-        b, c, t = x.shape
-        if t % self.period != 0:  # pad first
-            n_pad = self.period - (t % self.period)
-            x = F.pad(x, (0, n_pad), "reflect")
-            t = t + n_pad
-        x = x.view(b, c, t // self.period, self.period)
-
-        for l in self.convs:
-            x = l(x)
-            x = F.leaky_relu(x, LRELU_SLOPE)
-            fmap.append(x)
-        x = self.conv_post(x)
-        fmap.append(x)
-        x = torch.flatten(x, 1, -1)
-
-        return x, fmap
-
-
-class MultiPeriodDiscriminator(torch.nn.Module):
-    def __init__(self, periods=None):
-        super(MultiPeriodDiscriminator, self).__init__()
-        self.periods = periods if periods is not None else [2, 3, 5, 7, 11]
-        self.discriminators = nn.ModuleList()
-        for period in self.periods:
-            self.discriminators.append(DiscriminatorP(period))
-
-    def forward(self, y, y_hat):
-        y_d_rs = []
-        y_d_gs = []
-        fmap_rs = []
-        fmap_gs = []
-        for i, d in enumerate(self.discriminators):
-            y_d_r, fmap_r = d(y)
-            y_d_g, fmap_g = d(y_hat)
-            y_d_rs.append(y_d_r)
-            fmap_rs.append(fmap_r)
-            y_d_gs.append(y_d_g)
-            fmap_gs.append(fmap_g)
-
-        return y_d_rs, y_d_gs, fmap_rs, fmap_gs
-
-
-class DiscriminatorS(torch.nn.Module):
-    def __init__(self, use_spectral_norm=False):
-        super(DiscriminatorS, self).__init__()
-        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
-        self.convs = nn.ModuleList([
-            norm_f(Conv1d(1, 128, 15, 1, padding=7)),
-            norm_f(Conv1d(128, 128, 41, 2, groups=4, padding=20)),
-            norm_f(Conv1d(128, 256, 41, 2, groups=16, padding=20)),
-            norm_f(Conv1d(256, 512, 41, 4, groups=16, padding=20)),
-            norm_f(Conv1d(512, 1024, 41, 4, groups=16, padding=20)),
-            norm_f(Conv1d(1024, 1024, 41, 1, groups=16, padding=20)),
-            norm_f(Conv1d(1024, 1024, 5, 1, padding=2)),
-        ])
-        self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1))
-
-    def forward(self, x):
-        fmap = []
-        for l in self.convs:
-            x = l(x)
-            x = F.leaky_relu(x, LRELU_SLOPE)
-            fmap.append(x)
-        x = self.conv_post(x)
-        fmap.append(x)
-        x = torch.flatten(x, 1, -1)
-
-        return x, fmap
-
-
-class MultiScaleDiscriminator(torch.nn.Module):
-    def __init__(self):
-        super(MultiScaleDiscriminator, self).__init__()
-        self.discriminators = nn.ModuleList([
-            DiscriminatorS(use_spectral_norm=True),
-            DiscriminatorS(),
-            DiscriminatorS(),
-        ])
-        self.meanpools = nn.ModuleList([
-            AvgPool1d(4, 2, padding=2),
-            AvgPool1d(4, 2, padding=2)
-        ])
-
-    def forward(self, y, y_hat):
-        y_d_rs = []
-        y_d_gs = []
-        fmap_rs = []
-        fmap_gs = []
-        for i, d in enumerate(self.discriminators):
-            if i != 0:
-                y = self.meanpools[i - 1](y)
-                y_hat = self.meanpools[i - 1](y_hat)
-            y_d_r, fmap_r = d(y)
-            y_d_g, fmap_g = d(y_hat)
-            y_d_rs.append(y_d_r)
-            fmap_rs.append(fmap_r)
-            y_d_gs.append(y_d_g)
-            fmap_gs.append(fmap_g)
-
-        return y_d_rs, y_d_gs, fmap_rs, fmap_gs
-
-
-def feature_loss(fmap_r, fmap_g):
-    loss = 0
-    for dr, dg in zip(fmap_r, fmap_g):
-        for rl, gl in zip(dr, dg):
-            loss += torch.mean(torch.abs(rl - gl))
-
-    return loss * 2
-
-
-def discriminator_loss(disc_real_outputs, disc_generated_outputs):
-    loss = 0
-    r_losses = []
-    g_losses = []
-    for dr, dg in zip(disc_real_outputs, disc_generated_outputs):
-        r_loss = torch.mean((1 - dr) ** 2)
-        g_loss = torch.mean(dg ** 2)
-        loss += (r_loss + g_loss)
-        r_losses.append(r_loss.item())
-        g_losses.append(g_loss.item())
-
-    return loss, r_losses, g_losses
-
-
-def generator_loss(disc_outputs):
-    loss = 0
-    gen_losses = []
-    for dg in disc_outputs:
-        l = torch.mean((1 - dg) ** 2)
-        gen_losses.append(l)
-        loss += l
-
-    return loss, gen_losses

src/tts_vits/vdecoder/hifigan/nvSTFT.py

@@ -1,111 +0,0 @@
-import math
-import os
-os.environ["LRU_CACHE_CAPACITY"] = "3"
-import random
-import torch
-import torch.utils.data
-import numpy as np
-import librosa
-from librosa.util import normalize
-from librosa.filters import mel as librosa_mel_fn
-from scipy.io.wavfile import read
-import soundfile as sf
-
-def load_wav_to_torch(full_path, target_sr=None, return_empty_on_exception=False):
-    sampling_rate = None
-    try:
-        data, sampling_rate = sf.read(full_path, always_2d=True)# than soundfile.
-    except Exception as ex:
-        print(f"'{full_path}' failed to load.\nException:")
-        print(ex)
-        if return_empty_on_exception:
-            return [], sampling_rate or target_sr or 32000
-        else:
-            raise Exception(ex)
-    
-    if len(data.shape) > 1:
-        data = data[:, 0]
-        assert len(data) > 2# check duration of audio file is > 2 samples (because otherwise the slice operation was on the wrong dimension)
-    
-    if np.issubdtype(data.dtype, np.integer): # if audio data is type int
-        max_mag = -np.iinfo(data.dtype).min # maximum magnitude = min possible value of intXX
-    else: # if audio data is type fp32
-        max_mag = max(np.amax(data), -np.amin(data))
-        max_mag = (2**31)+1 if max_mag > (2**15) else ((2**15)+1 if max_mag > 1.01 else 1.0) # data should be either 16-bit INT, 32-bit INT or [-1 to 1] float32
-    
-    data = torch.FloatTensor(data.astype(np.float32))/max_mag
-    
-    if (torch.isinf(data) | torch.isnan(data)).any() and return_empty_on_exception:# resample will crash with inf/NaN inputs. return_empty_on_exception will return empty arr instead of except
-        return [], sampling_rate or target_sr or 32000
-    if target_sr is not None and sampling_rate != target_sr:
-        data = torch.from_numpy(librosa.core.resample(data.numpy(), orig_sr=sampling_rate, target_sr=target_sr))
-        sampling_rate = target_sr
-    
-    return data, sampling_rate
-
-def dynamic_range_compression(x, C=1, clip_val=1e-5):
-    return np.log(np.clip(x, a_min=clip_val, a_max=None) * C)
-
-def dynamic_range_decompression(x, C=1):
-    return np.exp(x) / C
-
-def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
-    return torch.log(torch.clamp(x, min=clip_val) * C)
-
-def dynamic_range_decompression_torch(x, C=1):
-    return torch.exp(x) / C
-
-class STFT():
-    def __init__(self, sr=22050, n_mels=80, n_fft=1024, win_size=1024, hop_length=256, fmin=20, fmax=11025, clip_val=1e-5):
-        self.target_sr = sr
-        
-        self.n_mels     = n_mels
-        self.n_fft      = n_fft
-        self.win_size   = win_size
-        self.hop_length = hop_length
-        self.fmin     = fmin
-        self.fmax     = fmax
-        self.clip_val = clip_val
-        self.mel_basis = {}
-        self.hann_window = {}
-    
-    def get_mel(self, y, center=False):
-        sampling_rate = self.target_sr
-        n_mels     = self.n_mels
-        n_fft      = self.n_fft
-        win_size   = self.win_size
-        hop_length = self.hop_length
-        fmin       = self.fmin
-        fmax       = self.fmax
-        clip_val   = self.clip_val
-        
-        if torch.min(y) < -1.:
-            print('min value is ', torch.min(y))
-        if torch.max(y) > 1.:
-            print('max value is ', torch.max(y))
-        
-        if fmax not in self.mel_basis:
-            mel = librosa_mel_fn(sr=sampling_rate, n_fft=n_fft, n_mels=n_mels, fmin=fmin, fmax=fmax)
-            self.mel_basis[str(fmax)+'_'+str(y.device)] = torch.from_numpy(mel).float().to(y.device)
-            self.hann_window[str(y.device)] = torch.hann_window(self.win_size).to(y.device)
-        
-        y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_length)/2), int((n_fft-hop_length)/2)), mode='reflect')
-        y = y.squeeze(1)
-        
-        spec = torch.stft(y, n_fft, hop_length=hop_length, win_length=win_size, window=self.hann_window[str(y.device)],
-                          center=center, pad_mode='reflect', normalized=False, onesided=True)
-        # print(111,spec)
-        spec = torch.sqrt(spec.pow(2).sum(-1)+(1e-9))
-        # print(222,spec)
-        spec = torch.matmul(self.mel_basis[str(fmax)+'_'+str(y.device)], spec)
-        # print(333,spec)
-        spec = dynamic_range_compression_torch(spec, clip_val=clip_val)
-        # print(444,spec)
-        return spec
-    
-    def __call__(self, audiopath):
-        audio, sr = load_wav_to_torch(audiopath, target_sr=self.target_sr)
-        spect = self.get_mel(audio.unsqueeze(0)).squeeze(0)
-        return spect
-
-stft = STFT()

src/tts_vits/vdecoder/hifigan/utils.py

@@ -1,68 +0,0 @@
-import glob
-import os
-import matplotlib
-import torch
-from torch.nn.utils import weight_norm
-matplotlib.use("Agg")
-import matplotlib.pylab as plt
-
-
-def plot_spectrogram(spectrogram):
-    fig, ax = plt.subplots(figsize=(10, 2))
-    im = ax.imshow(spectrogram, aspect="auto", origin="lower",
-                   interpolation='none')
-    plt.colorbar(im, ax=ax)
-
-    fig.canvas.draw()
-    plt.close()
-
-    return fig
-
-
-def init_weights(m, mean=0.0, std=0.01):
-    classname = m.__class__.__name__
-    if classname.find("Conv") != -1:
-        m.weight.data.normal_(mean, std)
-
-
-def apply_weight_norm(m):
-    classname = m.__class__.__name__
-    if classname.find("Conv") != -1:
-        weight_norm(m)
-
-
-def get_padding(kernel_size, dilation=1):
-    return int((kernel_size*dilation - dilation)/2)
-
-
-def load_checkpoint(filepath, device):
-    assert os.path.isfile(filepath)
-    print("Loading '{}'".format(filepath))
-    checkpoint_dict = torch.load(filepath, map_location=device)
-    print("Complete.")
-    return checkpoint_dict
-
-
-def save_checkpoint(filepath, obj):
-    print("Saving checkpoint to {}".format(filepath))
-    torch.save(obj, filepath)
-    print("Complete.")
-
-
-def del_old_checkpoints(cp_dir, prefix, n_models=2):
-    pattern = os.path.join(cp_dir, prefix + '????????')
-    cp_list = glob.glob(pattern) # get checkpoint paths
-    cp_list = sorted(cp_list)# sort by iter
-    if len(cp_list) > n_models: # if more than n_models models are found
-        for cp in cp_list[:-n_models]:# delete the oldest models other than lastest n_models
-            open(cp, 'w').close()# empty file contents
-            os.unlink(cp)# delete file (move to trash when using Colab)
-
-
-def scan_checkpoint(cp_dir, prefix):
-    pattern = os.path.join(cp_dir, prefix + '????????')
-    cp_list = glob.glob(pattern)
-    if len(cp_list) == 0:
-        return None
-    return sorted(cp_list)[-1]
-

src/tts_vits/vits_haruhi.py

@@ -1,51 +0,0 @@
-import requests
-import inference_main
-import time
-import uuid
-
-def set_model_path(path):
-    inference_main.set_model_path(path)
-
-def tts(text, spd):
-    url = f"https://fanyi.baidu.com/gettts?lan=jp&text={text}&spd={spd}&source=web"
-
-    payload = {}
-    headers = {
-    'Cookie': 'BAIDUID=543CBD0E4FB46C2FD5F44F7D81911F15:FG=1'
-    }
-
-    res = requests.request("GET", url, headers=headers, data=payload)
-    while res.content == b'':
-        res = requests.request("GET", url, headers=headers, data=payload)
-        time.sleep(0.1)
-   
-
-    if res.status_code == 200:
-        return res.content
-    else:
-        return None
-
-def vits_haruhi(text, tran, spd=3):
-    voice = tts(text, spd)
-    
-    if voice is None:
-        print("TTS failed")
-        return None
-    filename = f"tts_results/{str(uuid.uuid4())}.mp3";
-    with open(filename, "wb") as f:
-        f.write(voice)
-    return inference_main.infer_to("haruhi", tran, filename)
-
-
-if __name__ == "__main__":
-    inference_main.infer_tool.mkdir(["./tts_results"])
-    # 设置模型路径
-    set_model_path("vits_models/Haruhi_54000.pth")
-    # 生成语音
-    print( vits_haruhi("真実はいつもひとつ", 8))
-    print( vits_haruhi("私の青春は後悔していない", 8))
-    # vits_haruhi("またみんなで笑いたいのに君が死んだら意味が無いじゃないか！", 8)
-    # vits_haruhi("あきらめたらそこで試合終了だよ", 8)
-    # vits_haruhi("別れの味は分かりません。さようならという言葉がこんなに強いとは知りませんでした", 8)
-    # vits_haruhi("命には限りがあるからこそ、もっと大切に見える。命に限りがあるからこそ、たゆまぬ努力が必要だ", 8)
-    # vits_haruhi("なんとかなるよ！絶対大丈夫だよ", 8)