|
import logging |
|
logging.getLogger('numba').setLevel(logging.WARNING) |
|
logging.getLogger('matplotlib').setLevel(logging.WARNING) |
|
logging.getLogger('urllib3').setLevel(logging.WARNING) |
|
import romajitable |
|
import re |
|
import numpy as np |
|
import IPython.display as ipd |
|
import torch |
|
import commons |
|
import utils |
|
from models import SynthesizerTrn |
|
from text.symbols import symbols |
|
from text import text_to_sequence |
|
import gradio as gr |
|
import time |
|
import datetime |
|
import os |
|
import librosa |
|
from mel_processing import spectrogram_torch |
|
class VitsGradio: |
|
def __init__(self): |
|
self.dev = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") |
|
self.lan = ["中文","日文","自动","手动"] |
|
self.idols = ["c1","c2","高咲侑","歩夢","かすみ","しずく","果林","愛","彼方","せつ菜","璃奈","栞子","エマ","ランジュ","ミア","華恋","まひる","なな","クロディーヌ","ひかり",'純那',"香子","真矢","双葉","ミチル","メイファン","やちよ","晶","いちえ","ゆゆ子","塁","珠緒","あるる","ララフィン","美空","静羽","あるる"] |
|
self.modelPaths = [] |
|
for root,dirs,files in os.walk("checkpoints"): |
|
for dir in dirs: |
|
self.modelPaths.append(dir) |
|
with gr.Blocks() as self.Vits: |
|
gr.Markdown( |
|
"## <center> Lovelive虹团中日双语VITS\n" |
|
"### <center> 请不要生成会对个人以及企划造成侵害的内容\n" |
|
"<div align='center'>目前有标贝普通话版，去标贝版，少歌模型还是大饼状态</div>" |
|
'<div align="center"><a>参数说明:由于爱抖露们过于有感情，合成日语时建议将噪声比例调节至0.2-0.3区间，噪声偏差对应着每个字之间的间隔，对普通话影响较大，duration代表整体语速</div>' |
|
'<div align="center"><a>合成前请先选择模型，否则第一次合成不一定成功。长段落/小说合成建议colab或本地运行</div>') |
|
with gr.Tab("TTS合成"): |
|
with gr.Row(): |
|
with gr.Column(): |
|
with gr.Row(): |
|
with gr.Column(): |
|
input1 = gr.TextArea(label="Text", value="为什么你会那么熟练啊？你和雪菜亲过多少次了") |
|
input2 = gr.Dropdown(label="Language", choices=self.lan, value="自动", interactive=True) |
|
input3 = gr.Dropdown(label="Speaker", choices=self.idols, value="歩夢", interactive=True) |
|
btnVC = gr.Button("Submit") |
|
with gr.Column(): |
|
input4 = gr.Slider(minimum=0, maximum=1.0, label="更改噪声比例(noise scale)，以控制情感", value=0.267) |
|
input5 = gr.Slider(minimum=0, maximum=1.0, label="更改噪声偏差(noise scale w)，以控制音素长短", value=0.7) |
|
input6 = gr.Slider(minimum=0.1, maximum=10, label="duration", value=1) |
|
output1 = gr.Audio(label="采样率22050") |
|
btnVC.click(self.infer, inputs=[input1, input2, input3, input4, input5, input6], outputs=[output1]) |
|
with gr.Tab("选择模型"): |
|
with gr.Column(): |
|
modelstrs = gr.Dropdown(label = "模型", choices = self.modelPaths, value = self.modelPaths[0], type = "value") |
|
btnMod = gr.Button("载入模型") |
|
statusa = gr.TextArea() |
|
btnMod.click(self.loadCk, inputs=[modelstrs], outputs = [statusa]) |
|
with gr.Tab("Voice Conversion"): |
|
gr.Markdown(""" |
|
录制或上传声音，并选择要转换的音色。 |
|
""") |
|
with gr.Column(): |
|
record_audio = gr.Audio(label="record your voice", source="microphone") |
|
upload_audio = gr.Audio(label="or upload audio here", source="upload") |
|
source_speaker = gr.Dropdown(choices=self.idols, value="歩夢", label="source speaker") |
|
target_speaker = gr.Dropdown(choices=self.idols, value="歩夢", label="target speaker") |
|
with gr.Column(): |
|
message_box = gr.Textbox(label="Message") |
|
converted_audio = gr.Audio(label='converted audio') |
|
btn = gr.Button("Convert!") |
|
btn.click(self.vc_fn, inputs=[source_speaker, target_speaker, record_audio, upload_audio], |
|
outputs=[message_box, converted_audio]) |
|
with gr.Tab("小说合成(带字幕)"): |
|
with gr.Row(): |
|
with gr.Column(): |
|
with gr.Row(): |
|
with gr.Column(): |
|
input1 = gr.TextArea(label="建议colab或本地克隆后运行本仓库", value="为什么你会那么熟练啊？你和雪菜亲过多少次了") |
|
input2 = gr.Dropdown(label="Language", choices=self.lan, value="自动", interactive=True) |
|
input3 = gr.Dropdown(label="Speaker", choices=self.idols, value="歩夢", interactive=True) |
|
btnVC = gr.Button("Submit") |
|
with gr.Column(): |
|
input4 = gr.Slider(minimum=0, maximum=1.0, label="更改噪声比例(noise scale)，以控制情感", value=0.267) |
|
input5 = gr.Slider(minimum=0, maximum=1.0, label="更改噪声偏差(noise scale w)，以控制音素长短", value=0.7) |
|
input6 = gr.Slider(minimum=0.1, maximum=10, label="Duration", value=1) |
|
output1 = gr.Audio(label="采样率22050") |
|
subtitle = gr.outputs.File(label="字幕文件：subtitles.srt") |
|
btnVC.click(self.infer2, inputs=[input1, input2, input3, input4, input5, input6], outputs=[output1,subtitle]) |
|
|
|
def loadCk(self,path): |
|
self.hps = utils.get_hparams_from_file(f"checkpoints/{path}/config.json") |
|
self.net_g = SynthesizerTrn( |
|
len(symbols), |
|
self.hps.data.filter_length // 2 + 1, |
|
self.hps.train.segment_size // self.hps.data.hop_length, |
|
n_speakers=self.hps.data.n_speakers, |
|
**self.hps.model).to(self.dev) |
|
_ = self.net_g.eval() |
|
_ = utils.load_checkpoint(f"checkpoints/{path}/model.pth", self.net_g) |
|
return "success" |
|
|
|
def get_text(self,text): |
|
text_norm = text_to_sequence(text,self.hps.data.text_cleaners) |
|
if self.hps.data.add_blank: |
|
text_norm = commons.intersperse(text_norm, 0) |
|
text_norm = torch.LongTensor(text_norm) |
|
return text_norm |
|
|
|
def is_japanese(self,string): |
|
for ch in string: |
|
if ord(ch) > 0x3040 and ord(ch) < 0x30FF: |
|
return True |
|
return False |
|
|
|
def is_english(self,string): |
|
import re |
|
pattern = re.compile('^[A-Za-z0-9.,:;!?()_*"\' ]+$') |
|
if pattern.fullmatch(string): |
|
return True |
|
else: |
|
return False |
|
|
|
def selection(self,speaker): |
|
if speaker == "高咲侑": |
|
spk = 0 |
|
return spk |
|
|
|
elif speaker == "歩夢": |
|
spk = 1 |
|
return spk |
|
|
|
elif speaker == "かすみ": |
|
spk = 2 |
|
return spk |
|
|
|
elif speaker == "しずく": |
|
spk = 3 |
|
return spk |
|
|
|
elif speaker == "果林": |
|
spk = 4 |
|
return spk |
|
|
|
elif speaker == "愛": |
|
spk = 5 |
|
return spk |
|
|
|
elif speaker == "彼方": |
|
spk = 6 |
|
return spk |
|
|
|
elif speaker == "せつ菜": |
|
spk = 7 |
|
return spk |
|
elif speaker == "エマ": |
|
spk = 8 |
|
return spk |
|
elif speaker == "璃奈": |
|
spk = 9 |
|
return spk |
|
elif speaker == "栞子": |
|
spk = 10 |
|
return spk |
|
elif speaker == "ランジュ": |
|
spk = 11 |
|
return spk |
|
elif speaker == "ミア": |
|
spk = 12 |
|
return spk |
|
|
|
elif speaker == "派蒙": |
|
spk = 16 |
|
return spk |
|
|
|
elif speaker == "c1": |
|
spk = 18 |
|
return spk |
|
|
|
elif speaker == "c2": |
|
spk = 19 |
|
return spk |
|
|
|
elif speaker == "華恋": |
|
spk = 21 |
|
return spk |
|
|
|
elif speaker == "まひる": |
|
spk = 22 |
|
return spk |
|
|
|
elif speaker == "なな": |
|
spk = 23 |
|
return spk |
|
|
|
elif speaker == "クロディーヌ": |
|
spk = 24 |
|
return spk |
|
|
|
elif speaker == "ひかり": |
|
spk = 25 |
|
return spk |
|
|
|
elif speaker == "純那": |
|
spk = 26 |
|
return spk |
|
|
|
elif speaker == "香子": |
|
spk = 27 |
|
return spk |
|
|
|
elif speaker == "真矢": |
|
spk = 28 |
|
return spk |
|
elif speaker == "双葉": |
|
spk = 29 |
|
return spk |
|
elif speaker == "ミチル": |
|
spk = 30 |
|
return spk |
|
elif speaker == "メイファン": |
|
spk = 31 |
|
return spk |
|
elif speaker == "やちよ": |
|
spk = 32 |
|
return spk |
|
elif speaker == "晶": |
|
spk = 33 |
|
return spk |
|
elif speaker == "いちえ": |
|
spk = 34 |
|
return spk |
|
elif speaker == "ゆゆ子": |
|
spk = 35 |
|
return spk |
|
elif speaker == "塁": |
|
spk = 36 |
|
return spk |
|
elif speaker == "珠緒": |
|
spk = 37 |
|
return spk |
|
elif speaker == "あるる": |
|
spk = 38 |
|
return spk |
|
elif speaker == "ララフィン": |
|
spk = 39 |
|
return spk |
|
elif speaker == "美空": |
|
spk = 40 |
|
return spk |
|
elif speaker == "静羽": |
|
spk = 41 |
|
return spk |
|
else: |
|
return 0 |
|
|
|
|
|
def sle(self,language,text): |
|
text = text.replace('\n','。').replace(' ',',') |
|
if language == "中文": |
|
tts_input1 = "[ZH]" + text + "[ZH]" |
|
return tts_input1 |
|
elif language == "自动": |
|
tts_input1 = f"[JA]{text}[JA]" if self.is_japanese(text) else f"[ZH]{text}[ZH]" |
|
return tts_input1 |
|
elif language == "日文": |
|
tts_input1 = "[JA]" + text + "[JA]" |
|
return tts_input1 |
|
elif language == "英文": |
|
tts_input1 = "[EN]" + text + "[EN]" |
|
return tts_input1 |
|
elif language == "手动": |
|
return text |
|
|
|
def extrac(self,text): |
|
text = re.sub("<[^>]*>","",text) |
|
result_list = re.split(r'\n', text) |
|
final_list = [] |
|
for i in result_list: |
|
if self.is_english(i): |
|
i = romajitable.to_kana(i).katakana |
|
i = i.replace('\n','').replace(' ','') |
|
|
|
if len(i)>1: |
|
if len(i) > 20: |
|
try: |
|
cur_list = re.split(r'。|！', i) |
|
for i in cur_list: |
|
if len(i)>1: |
|
final_list.append(i+'。') |
|
except: |
|
pass |
|
else: |
|
final_list.append(i) |
|
final_list = [x for x in final_list if x != ''] |
|
print(final_list) |
|
return final_list |
|
|
|
def vc_fn(self,original_speaker, target_speaker, record_audio, upload_audio): |
|
input_audio = record_audio if record_audio is not None else upload_audio |
|
if input_audio is None: |
|
return "You need to record or upload an audio", None |
|
sampling_rate, audio = input_audio |
|
original_speaker_id = self.selection(original_speaker) |
|
target_speaker_id = self.selection(target_speaker) |
|
|
|
audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32) |
|
if len(audio.shape) > 1: |
|
audio = librosa.to_mono(audio.transpose(1, 0)) |
|
if sampling_rate != self.hps.data.sampling_rate: |
|
audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=self.hps.data.sampling_rate) |
|
with torch.no_grad(): |
|
y = torch.FloatTensor(audio) |
|
y = y / max(-y.min(), y.max()) / 0.99 |
|
y = y.to(self.dev) |
|
y = y.unsqueeze(0) |
|
spec = spectrogram_torch(y, self.hps.data.filter_length, |
|
self.hps.data.sampling_rate, self.hps.data.hop_length, self.hps.data.win_length, |
|
center=False).to(self.dev) |
|
spec_lengths = torch.LongTensor([spec.size(-1)]).to(self.dev) |
|
sid_src = torch.LongTensor([original_speaker_id]).to(self.dev) |
|
sid_tgt = torch.LongTensor([target_speaker_id]).to(self.dev) |
|
audio = self.net_g.voice_conversion(spec, spec_lengths, sid_src=sid_src, sid_tgt=sid_tgt)[0][ |
|
0, 0].data.cpu().float().numpy() |
|
del y, spec, spec_lengths, sid_src, sid_tgt |
|
return "Success", (self.hps.data.sampling_rate, audio) |
|
|
|
def infer(self, text ,language, speaker_id,n_scale= 0.667,n_scale_w = 0.8, l_scale = 1): |
|
try: |
|
speaker_id = int(self.selection(speaker_id)) |
|
t1 = time.time() |
|
stn_tst = self.get_text(self.sle(language,text)) |
|
with torch.no_grad(): |
|
x_tst = stn_tst.unsqueeze(0).to(self.dev) |
|
x_tst_lengths = torch.LongTensor([stn_tst.size(0)]).to(self.dev) |
|
sid = torch.LongTensor([speaker_id]).to(self.dev) |
|
audio = self.net_g.infer(x_tst, x_tst_lengths, sid=sid, noise_scale=n_scale, noise_scale_w=n_scale_w, length_scale=l_scale)[0][0,0].data.cpu().float().numpy() |
|
t2 = time.time() |
|
spending_time = "推理时间为："+str(t2-t1)+"s" |
|
print(spending_time) |
|
return (self.hps.data.sampling_rate, audio) |
|
except: |
|
self.hps = utils.get_hparams_from_file(f"checkpoints/biaobei/config.json") |
|
self.net_g = SynthesizerTrn( |
|
len(symbols), |
|
self.hps.data.filter_length // 2 + 1, |
|
self.hps.train.segment_size // self.hps.data.hop_length, |
|
n_speakers=self.hps.data.n_speakers, |
|
**self.hps.model).to(self.dev) |
|
_ = self.net_g.eval() |
|
_ = utils.load_checkpoint(f"checkpoints/biaobei/model.pth", self.net_g) |
|
|
|
def infer2(self, text ,language, speaker_id,n_scale= 0.667,n_scale_w = 0.8, l_scale = 1): |
|
speaker_id = int(self.selection(speaker_id)) |
|
a = ['【','[','(','（'] |
|
b = ['】',']',')','）'] |
|
for i in a: |
|
text = text.replace(i,'<') |
|
for i in b: |
|
text = text.replace(i,'>') |
|
final_list = self.extrac(text.replace('“','').replace('”','')) |
|
audio_fin = [] |
|
c = 0 |
|
t = datetime.timedelta(seconds=0) |
|
f1 = open("subtitles.srt",'w',encoding='utf-8') |
|
for sentence in final_list: |
|
c +=1 |
|
stn_tst = self.get_text(self.sle(language,sentence)) |
|
with torch.no_grad(): |
|
x_tst = stn_tst.unsqueeze(0).to(self.dev) |
|
x_tst_lengths = torch.LongTensor([stn_tst.size(0)]).to(self.dev) |
|
sid = torch.LongTensor([speaker_id]).to(self.dev) |
|
t1 = time.time() |
|
audio = self.net_g.infer(x_tst, x_tst_lengths, sid=sid, noise_scale=n_scale, noise_scale_w=n_scale_w, length_scale=l_scale)[0][0,0].data.cpu().float().numpy() |
|
t2 = time.time() |
|
spending_time = "第"+str(c)+"句的推理时间为："+str(t2-t1)+"s" |
|
print(spending_time) |
|
time_start = str(t).split(".")[0] + "," + str(t.microseconds)[:3] |
|
last_time = datetime.timedelta(seconds=len(audio)/float(22050)) |
|
t+=last_time |
|
time_end = str(t).split(".")[0] + "," + str(t.microseconds)[:3] |
|
print(time_end) |
|
f1.write(str(c-1)+'\n'+time_start+' --> '+time_end+'\n'+sentence+'\n\n') |
|
audio_fin.append(audio) |
|
file_path = "subtitles.srt" |
|
return (self.hps.data.sampling_rate, np.concatenate(audio_fin)),file_path |
|
print("开始部署") |
|
grVits = VitsGradio() |
|
grVits.Vits.launch() |
