app.py

import logging
logging.getLogger('numba').setLevel(logging.WARNING)
logging.getLogger('matplotlib').setLevel(logging.WARNING)
logging.getLogger('urllib3').setLevel(logging.WARNING)
import romajitable
import re
import numpy as np
import IPython.display as ipd
import torch
import commons
import utils
from models import SynthesizerTrn
from text.symbols import symbols
from text import text_to_sequence
import gradio as gr
import time
import datetime
import os
import librosa
from mel_processing import spectrogram_torch
class VitsGradio:
    def __init__(self):
        self.dev = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
        self.lan = ["中文","日文","自动","手动"]
        self.idols = ["chinese1","chinese2","chinese3","高咲侑","歩夢","かすみ","しずく","果林","愛","彼方","せつ菜","璃奈","栞子","エマ","ランジュ","ミア","華恋","まひる","なな","クロディーヌ","ひかり",'純那',"香子","真矢","双葉","ミチル","メイファン","やちよ","晶","いちえ","ゆゆ子","塁","珠緒","あるる","ララフィン","美空","静羽","あるる"]
        self.modelPaths = []
        for root,dirs,files in os.walk("checkpoints"):
            for dir in dirs:
                self.modelPaths.append(dir)
        with gr.Blocks() as self.Vits:
            gr.Markdown(
            "## <center> Lovelive虹团中日双语VITS\n"
            "### <center> 请不要生成会对个人以及企划造成侵害的内容\n"
            "<div align='center'>目前有虹团标贝普通话版(biaobei)，虹团模型(default)，少歌模型(ShojoKageki)以及混合模型(tmp)</div>"
            '<div align="center"><a>参数说明:默认参数适合汉语普通话，合成日语时建议将噪声比例调节至0.667，噪声偏差对应着每个字之间的间隔，对普通话影响较大，duration代表整体语速</div>'
            '<div align="center"><a>合成前请先选择模型，建议选择tmp模型，否则第一次合成不一定成功。长段落/小说合成建议colab或本地运行</div>')
            with gr.Tab("TTS合成"):
                with gr.Row():
                    with gr.Column():
                        with gr.Row():
                            with gr.Column():
                                input1 = gr.TextArea(label="Text", value="为什么你会那么熟练啊？你和雪菜亲过多少次了")
                                input2 = gr.Dropdown(label="Language", choices=self.lan, value="自动", interactive=True)
                                input3 = gr.Dropdown(label="Speaker", choices=self.idols, value="歩夢", interactive=True)
                                btnVC = gr.Button("Submit")
                            with gr.Column():
                                input4 = gr.Slider(minimum=0, maximum=1.0, label="更改噪声比例(noise scale)，以控制情感", value=0.267)
                                input5 = gr.Slider(minimum=0, maximum=1.0, label="更改噪声偏差(noise scale w)，以控制音素长短", value=0.7)
                                input6 = gr.Slider(minimum=0.1, maximum=10, label="duration", value=1)
                                output1 = gr.Audio(label="采样率22050")
                btnVC.click(self.infer, inputs=[input1, input2, input3, input4, input5, input6], outputs=[output1])
            with gr.Tab("选择模型"):
                with gr.Column():
                    modelstrs = gr.Dropdown(label = "模型", choices = self.modelPaths, value = self.modelPaths[0], type = "value")
                    btnMod = gr.Button("载入模型")
                    statusa = gr.TextArea()
                    btnMod.click(self.loadCk, inputs=[modelstrs], outputs = [statusa])
            with gr.Tab("Voice Conversion"):
                gr.Markdown("""
                                录制或上传声音，并选择要转换的音色。
                """)
                with gr.Column():
                    record_audio = gr.Audio(label="record your voice", source="microphone")
                    upload_audio = gr.Audio(label="or upload audio here", source="upload")
                    source_speaker = gr.Dropdown(choices=self.idols, value="歩夢", label="source speaker")
                    target_speaker = gr.Dropdown(choices=self.idols, value="歩夢", label="target speaker")
                with gr.Column():
                    message_box = gr.Textbox(label="Message")
                    converted_audio = gr.Audio(label='converted audio')
                btn = gr.Button("Convert!")
                btn.click(self.vc_fn, inputs=[source_speaker, target_speaker, record_audio, upload_audio],
                        outputs=[message_box, converted_audio])
            with gr.Tab("小说合成(带字幕)"):
                with gr.Row():
                    with gr.Column():
                        with gr.Row():
                            with gr.Column():
                                input1 = gr.TextArea(label="建议colab或本地克隆后运行本仓库", value="为什么你会那么熟练啊？你和雪菜亲过多少次了")
                                input2 = gr.Dropdown(label="Language", choices=self.lan, value="自动", interactive=True)
                                input3 = gr.Dropdown(label="Speaker", choices=self.idols, value="歩夢", interactive=True)
                                btnVC = gr.Button("Submit")
                            with gr.Column():
                                input4 = gr.Slider(minimum=0, maximum=1.0, label="更改噪声比例(noise scale)，以控制情感", value=0.267)
                                input5 = gr.Slider(minimum=0, maximum=1.0, label="更改噪声偏差(noise scale w)，以控制音素长短", value=0.7)
                                input6 = gr.Slider(minimum=0.1, maximum=10, label="Duration", value=1)
                                output1 = gr.Audio(label="采样率22050")
                                subtitle = gr.outputs.File(label="字幕文件：subtitles.srt")
                btnVC.click(self.infer2, inputs=[input1, input2, input3, input4, input5, input6], outputs=[output1,subtitle])
    
    def loadCk(self,path):
        self.hps = utils.get_hparams_from_file(f"checkpoints/{path}/config.json")
        self.net_g = SynthesizerTrn(
            len(symbols),
            self.hps.data.filter_length // 2 + 1,
            self.hps.train.segment_size // self.hps.data.hop_length,
            n_speakers=self.hps.data.n_speakers,
            **self.hps.model).to(self.dev)
        _ = self.net_g.eval()
        _ = utils.load_checkpoint(f"checkpoints/{path}/model.pth", self.net_g)
        return "success"

    def get_text(self,text):
        text_norm = text_to_sequence(text,self.hps.data.text_cleaners)
        if self.hps.data.add_blank:
            text_norm = commons.intersperse(text_norm, 0)
        text_norm = torch.LongTensor(text_norm)
        return text_norm
    
    def is_japanese(self,string):
        for ch in string:
            if ord(ch) > 0x3040 and ord(ch) < 0x30FF:
                return True
        return False
    
    def is_english(self,string):
        import re
        pattern = re.compile('^[A-Za-z0-9.,:;!?()_*"\' ]+$')
        if pattern.fullmatch(string):
            return True
        else:
            return False
    
    def selection(self,speaker):
        if speaker == "高咲侑":
            spk = 0
            return spk

        elif speaker == "歩夢":
            spk = 1
            return spk

        elif speaker == "かすみ":
            spk = 2
            return spk

        elif speaker == "しずく":
            spk = 3
            return spk

        elif speaker == "果林":
            spk = 4
            return spk
    
        elif speaker == "愛":
            spk = 5
            return spk

        elif speaker == "彼方":
            spk = 6
            return spk

        elif speaker == "せつ菜":
            spk = 7
            return spk
        elif speaker == "エマ":
            spk = 8
            return spk
        elif speaker == "璃奈":
            spk = 9
            return spk
        elif speaker == "栞子":
            spk = 10
            return spk
        elif speaker == "ランジュ":
            spk = 11
            return spk
        elif speaker == "ミア":
            spk = 12
            return spk
        
        elif speaker == "chinese1":
            spk = 16
            return spk
        
        elif speaker == "chinese2":
            spk = 18
            return spk

        elif speaker == "chinese3":
            spk = 19
            return spk

        elif speaker == "華恋":
            spk = 21
            return spk

        elif speaker == "まひる":
            spk = 22
            return spk
    
        elif speaker == "なな":
            spk = 23
            return spk
    
        elif speaker == "クロディーヌ":
            spk = 24
            return spk
    
        elif speaker == "ひかり":
            spk = 25
            return spk
    
        elif speaker == "純那":
            spk = 26
            return spk
    
        elif speaker == "香子":
            spk = 27
            return spk
    
        elif speaker == "真矢":
            spk = 28
            return spk
        elif speaker == "双葉":
            spk = 29
            return spk
        elif speaker == "ミチル":
            spk = 30
            return spk
        elif speaker == "メイファン":
            spk = 31
            return spk
        elif speaker == "やちよ":
            spk = 32
            return spk
        elif speaker == "晶":
            spk = 33
            return spk
        elif speaker == "いちえ":
            spk = 34
            return spk
        elif speaker == "ゆゆ子":
            spk = 35
            return spk
        elif speaker == "塁":
            spk = 36
            return spk
        elif speaker == "珠緒":
            spk = 37
            return spk
        elif speaker == "あるる":
            spk = 38
            return spk
        elif speaker == "ララフィン":
            spk = 39
            return spk
        elif speaker == "美空":
            spk = 40
            return spk
        elif speaker == "静羽":
            spk = 41
            return spk
        else:
            return 0
            
    
    def sle(self,language,text):
        text = text.replace('\n','。').replace(' ',',')
        if language == "中文":
            tts_input1 = "[ZH]" + text + "[ZH]"
            return tts_input1
        elif language == "自动":
            tts_input1 = f"[JA]{text}[JA]" if self.is_japanese(text) else f"[ZH]{text}[ZH]"
            return tts_input1
        elif language == "日文":
            tts_input1 = "[JA]" + text + "[JA]"
            return tts_input1
        elif language == "英文":
            tts_input1 = "[EN]" + text + "[EN]"
            return tts_input1
        elif language == "手动":
            return text
    
    def extrac(self,text):
        text = re.sub("<[^>]*>","",text)
        result_list = re.split(r'\n', text)
        final_list = []
        for i in result_list:
            if self.is_english(i):
                i = romajitable.to_kana(i).katakana
            i = i.replace('\n','').replace(' ','')
            #Current length of single sentence: 20 
            '''
            if len(i)>1:
                if len(i) > 20:
                    try:
                        cur_list = re.split(r'。|！', i)
                        for i in cur_list:
                            if len(i)>1:
                                final_list.append(i+'。')
                    except:
                        pass
                else:
                    final_list.append(i)
            '''
            try:
                final_list.append(i)
            except:
                pass
        final_list = [x for x in final_list if x != '']
        print(final_list)
        return final_list
    
    def vc_fn(self,original_speaker, target_speaker, record_audio, upload_audio):
        input_audio = record_audio if record_audio is not None else upload_audio
        if input_audio is None:
            return "You need to record or upload an audio", None
        sampling_rate, audio = input_audio
        original_speaker_id = self.selection(original_speaker)
        target_speaker_id = self.selection(target_speaker)

        audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32)
        if len(audio.shape) > 1:
            audio = librosa.to_mono(audio.transpose(1, 0))
        if sampling_rate != self.hps.data.sampling_rate:
            audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=self.hps.data.sampling_rate)
        with torch.no_grad():
            y = torch.FloatTensor(audio)
            y = y / max(-y.min(), y.max()) / 0.99
            y = y.to(self.dev)
            y = y.unsqueeze(0)
            spec = spectrogram_torch(y, self.hps.data.filter_length,
                                     self.hps.data.sampling_rate, self.hps.data.hop_length, self.hps.data.win_length,
                                     center=False).to(self.dev)
            spec_lengths = torch.LongTensor([spec.size(-1)]).to(self.dev)
            sid_src = torch.LongTensor([original_speaker_id]).to(self.dev)
            sid_tgt = torch.LongTensor([target_speaker_id]).to(self.dev)
            audio = self.net_g.voice_conversion(spec, spec_lengths, sid_src=sid_src, sid_tgt=sid_tgt)[0][
                0, 0].data.cpu().float().numpy()
        del y, spec, spec_lengths, sid_src, sid_tgt
        return "Success", (self.hps.data.sampling_rate, audio)
    
    def infer(self, text ,language, speaker_id,n_scale= 0.667,n_scale_w = 0.8, l_scale = 1):
        try:
            speaker_id = int(self.selection(speaker_id))
            t1 = time.time()
            stn_tst = self.get_text(self.sle(language,text))
            with torch.no_grad():
                x_tst = stn_tst.unsqueeze(0).to(self.dev)
                x_tst_lengths = torch.LongTensor([stn_tst.size(0)]).to(self.dev)
                sid = torch.LongTensor([speaker_id]).to(self.dev)
                audio = self.net_g.infer(x_tst, x_tst_lengths, sid=sid, noise_scale=n_scale, noise_scale_w=n_scale_w, length_scale=l_scale)[0][0,0].data.cpu().float().numpy()
                t2 = time.time()
                spending_time = "推理时间为："+str(t2-t1)+"s"
                print(spending_time)
            return (self.hps.data.sampling_rate, audio)
        except:
            self.hps = utils.get_hparams_from_file(f"checkpoints/biaobei/config.json")
            self.net_g = SynthesizerTrn(
                len(symbols),
                self.hps.data.filter_length // 2 + 1,
                self.hps.train.segment_size // self.hps.data.hop_length,
                n_speakers=self.hps.data.n_speakers,
                **self.hps.model).to(self.dev)
            _ = self.net_g.eval()
            _ = utils.load_checkpoint(f"checkpoints/biaobei/model.pth", self.net_g)

    def infer2(self, text ,language, speaker_id,n_scale= 0.667,n_scale_w = 0.8, l_scale = 1):
        speaker_id = int(self.selection(speaker_id))
        a = ['【','[','(','（']
        b = ['】',']',')','）']
        for i in a:
            text = text.replace(i,'<')
        for i in b:
            text = text.replace(i,'>')
        final_list = self.extrac(text.replace('“','').replace('”',''))
        audio_fin = []
        c = 0
        t = datetime.timedelta(seconds=0)
        f1 = open("subtitles.srt",'w',encoding='utf-8')
        for sentence in final_list:
            c +=1
            stn_tst = self.get_text(self.sle(language,sentence))
            with torch.no_grad():
                x_tst = stn_tst.unsqueeze(0).to(self.dev)
                x_tst_lengths = torch.LongTensor([stn_tst.size(0)]).to(self.dev)
                sid = torch.LongTensor([speaker_id]).to(self.dev)
                t1 = time.time()
                audio = self.net_g.infer(x_tst, x_tst_lengths, sid=sid, noise_scale=n_scale, noise_scale_w=n_scale_w, length_scale=l_scale)[0][0,0].data.cpu().float().numpy()
                t2 = time.time()
                spending_time = "第"+str(c)+"句的推理时间为："+str(t2-t1)+"s"
                print(spending_time)
                time_start = str(t).split(".")[0] + "," + str(t.microseconds)[:3]
                last_time = datetime.timedelta(seconds=len(audio)/float(22050))
                t+=last_time
                time_end = str(t).split(".")[0] + "," + str(t.microseconds)[:3]
                print(time_end)
                f1.write(str(c-1)+'\n'+time_start+' --> '+time_end+'\n'+sentence+'\n\n')
                audio_fin.append(audio)
        file_path = "subtitles.srt"
        return (self.hps.data.sampling_rate, np.concatenate(audio_fin)),file_path
print("开始部署")
grVits = VitsGradio()
grVits.Vits.launch()