|
import torch |
|
import scipy |
|
import os |
|
import streamlit as st |
|
import pandas as pd |
|
from transformers import pipeline |
|
from transformers import VitsTokenizer, VitsModel |
|
from datasets import load_dataset, Audio |
|
from huggingface_hub.inference_api import InferenceApi |
|
|
|
from src import * |
|
|
|
|
|
|
|
col1, col2 = st.columns([20,3]) |
|
with col2: |
|
st.image('logo.png', use_column_width=True) |
|
with col1: |
|
st.title("Mockingbird") |
|
st.header("A demo of open Text to Speech tools") |
|
|
|
tts, about = st.tabs(["Text to speech", "**About**"]) |
|
|
|
|
|
with tts: |
|
|
|
|
|
tts_lang = st.selectbox('Language of text', (language_list), format_func = decode_iso) |
|
tts_text = st.text_area(label = "Please enter your sentence here:", |
|
value="", placeholder=placeholders[tts_lang] ) |
|
|
|
target_speaker_file = st.file_uploader("If you would like to test voice conversion, you may upload your audio below. You should upload one file in .wav format. If you don't, a default file will be used.", |
|
type=['wav']) |
|
|
|
|
|
if st.button("Generate"): |
|
|
|
|
|
if tts_lang in ['rus', 'fas']: |
|
st.warning("WARNING! On Windows, ESpeak-NG has trouble synthesizing output when input is provided from non-Latin alphabets.") |
|
|
|
st.divider() |
|
|
|
|
|
with st.spinner(":rainbow[Synthesizing, please wait... (this will be slowest the first time you generate audio in a new language)]"): |
|
if tts_text == "": |
|
tts_text=placeholders[tts_lang] |
|
|
|
|
|
base_mms = synth_mms(tts_text, models[tts_lang]['mms']) |
|
base_coqui= synth_coqui(tts_text, models[tts_lang]['coqui']) |
|
base_espeakng= synth_espeakng(tts_text, models[tts_lang]['espeakng']) |
|
base_toucan= synth_toucan(tts_text, models[tts_lang]['toucan']) |
|
|
|
if tts_lang=="swh": |
|
finetuned_mms1 = synth_mms(tts_text, "khof312/mms-tts-swh-female-1") |
|
finetuned_mms2 = synth_mms(tts_text, "khof312/mms-tts-swh-female-2") |
|
if tts_lang=="spa": |
|
finetuned_mms1 = synth_mms(tts_text, "ylacombe/mms-spa-finetuned-argentinian-monospeaker") |
|
finetuned_mms2 = synth_mms(tts_text, "ylacombe/mms-spa-finetuned-chilean-monospeaker") |
|
finetuned_mms3 = synth_mms(tts_text, "ylacombe/mms-spa-finetuned-colombian-monospeaker") |
|
|
|
|
|
|
|
|
|
"## Synthesis" |
|
"### Default models" |
|
row1 = st.columns([1,1,2]) |
|
row2 = st.columns([1,1,2]) |
|
row3 = st.columns([1,1,2]) |
|
row4 = st.columns([1,1,2]) |
|
row5 = st.columns([1,1,2]) |
|
|
|
row1[0].write("**Model**") |
|
row1[1].write("**Configuration**") |
|
row1[2].write("**Audio**") |
|
|
|
if base_mms is not None: |
|
row2[0].write(f"[Meta MMS](https://huggingface.co/docs/transformers/main/en/model_doc/mms)") |
|
row2[1].write("default") |
|
row2[2].audio(base_mms[0], sample_rate = base_mms[1]) |
|
|
|
if base_coqui is not None: |
|
row3[0].write(f"[Coqui](https://docs.coqui.ai/en/latest/index.html)") |
|
row3[1].write("default") |
|
row3[2].audio(base_coqui[0], sample_rate = base_coqui[1]) |
|
|
|
if base_espeakng is not None: |
|
row4[0].write(f"[Espeak-ng](https://github.com/espeak-ng/espeak-ng)") |
|
row4[1].write("default") |
|
row4[2].audio(base_espeakng[0], sample_rate = base_espeakng[1]) |
|
|
|
|
|
row5[0].write(f"[IMS-Toucan](https://github.com/DigitalPhonetics/IMS-Toucan)") |
|
row5[1].write("default") |
|
row5[2].audio(base_toucan[0], sample_rate = base_toucan[1]) |
|
|
|
|
|
|
|
if tts_lang == "swh": |
|
"### Fine Tuned" |
|
row1 = st.columns([1,1,2]) |
|
row2 = st.columns([1,1,2]) |
|
row3 = st.columns([1,1,2]) |
|
|
|
row1[0].write("**Model**") |
|
row1[1].write("**Configuration**") |
|
row1[2].write("**Audio**") |
|
|
|
row2[0].write(f"Meta MMS") |
|
row2[1].write("[female 1](https://huggingface.co/khof312/mms-tts-swh-female-1)") |
|
row2[2].audio(finetuned_mms1[0], sample_rate = finetuned_mms1[1]) |
|
row3[0].write(f"Meta MMS") |
|
row3[1].write("[female 2](https://huggingface.co/khof312/mms-tts-swh-female-2)") |
|
row3[2].audio(finetuned_mms2[0], sample_rate = finetuned_mms2[1]) |
|
|
|
|
|
if tts_lang == "spa": |
|
"### Fine Tuned" |
|
row1 = st.columns([1,1,2]) |
|
row2 = st.columns([1,1,2]) |
|
row3 = st.columns([1,1,2]) |
|
row4 = st.columns([1,1,2]) |
|
|
|
row1[0].write("**Model**") |
|
row1[1].write("**Configuration**") |
|
row1[2].write("**Audio**") |
|
|
|
row2[0].write(f"Meta MMS") |
|
row2[1].write("[ylacombe - Argentinian](https://huggingface.co/ylacombe/mms-spa-finetuned-argentinian-monospeaker)") |
|
row2[2].audio(finetuned_mms1[0], sample_rate = finetuned_mms1[1]) |
|
row3[0].write(f"Meta MMS") |
|
row3[1].write("[ylacombe - Chilean](https://huggingface.co/ylacombe/mms-spa-finetuned-chilean-monospeaker)") |
|
row3[2].audio(finetuned_mms2[0], sample_rate = finetuned_mms2[1]) |
|
row4[0].write(f"Meta MMS") |
|
row4[1].write("[ylacombe - Colombian](https://huggingface.co/ylacombe/mms-spa-finetuned-colombian-monospeaker)") |
|
row4[2].audio(finetuned_mms3[0], sample_rate = finetuned_mms3[1]) |
|
|
|
st.divider() |
|
|
|
"## Voice conversion" |
|
|
|
|
|
|
|
st.warning('''Note: The naturalness of the audio will only be as good as that of the audio in "default models" above.''') |
|
|
|
if target_speaker_file is not None: |
|
rate, wav = scipy.io.wavfile.read(target_speaker_file) |
|
scipy.io.wavfile.write("target_speaker_custom.wav", data=wav, rate=rate) |
|
target_speaker = "target_speaker_custom.wav" |
|
else: |
|
target_speaker = "target_speaker.wav" |
|
|
|
if base_mms is not None: |
|
scipy.io.wavfile.write("source_speaker_mms.wav", rate=base_mms[1], data=base_mms[0].T) |
|
converted_mms = convert_coqui('source_speaker_mms.wav', target_speaker) |
|
|
|
if base_coqui is not None: |
|
scipy.io.wavfile.write("source_speaker_coqui.wav", rate=base_coqui[1], data=base_coqui[0].T) |
|
converted_coqui = convert_coqui('source_speaker_coqui.wav', target_speaker) |
|
|
|
if base_espeakng is not None: |
|
scipy.io.wavfile.write("source_speaker_espeakng.wav", rate=base_espeakng[1], data=base_espeakng[0].T) |
|
converted_espeakng = convert_coqui('source_speaker_espeakng.wav', target_speaker) |
|
|
|
scipy.io.wavfile.write("source_speaker_toucan.wav", rate=base_toucan[1], data=base_toucan[0].T) |
|
converted_toucan = convert_coqui('source_speaker_toucan.wav', target_speaker) |
|
|
|
row1 = st.columns([1,1,2]) |
|
row2 = st.columns([1,1,2]) |
|
row3 = st.columns([1,1,2]) |
|
row4 = st.columns([1,1,2]) |
|
|
|
row1[0].write("**Model**") |
|
row1[1].write("**Configuration**") |
|
row1[2].write("**Audio**") |
|
|
|
if base_mms is not None: |
|
row1[0].write(f"Meta MMS") |
|
row1[1].write(f"converted") |
|
row1[2].audio(converted_mms[0], sample_rate = converted_mms[1]) |
|
|
|
if base_coqui is not None: |
|
row2[0].write(f"Coqui") |
|
row2[1].write(f"converted") |
|
row2[2].audio(converted_coqui[0], sample_rate = converted_coqui[1]) |
|
|
|
if base_espeakng is not None: |
|
row3[0].write(f"Espeak-ng") |
|
row3[1].write(f"converted") |
|
row3[2].audio(converted_espeakng[0], sample_rate = converted_espeakng[1]) |
|
|
|
|
|
row4[0].write(f"IMS Toucan") |
|
row4[1].write(f"converted") |
|
row4[2].audio(converted_toucan[0], sample_rate = converted_toucan[1]) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
with about: |
|
|
|
st.markdown('''# Mockingbird TTS Demo |
|
This page is a demo of the openly available Text to Speech models for various languages of interest. Currently, 4 synthesizers are supported: |
|
- [**Meta's Massively Multilingual Speech (MMS)**](https://ai.meta.com/blog/multilingual-model-speech-recognition/) model, which supports over 1000 languages.[^1] |
|
- [**Coqui's TTS**](https://docs.coqui.ai/en/latest/#) package;[^2] while no longer supported, Coqui acted as a hub for TTS model hosting and these models are still available. |
|
- [**ESpeak-NG's**](https://github.com/espeak-ng/espeak-ng/tree/master)'s synthetic voices**[^3] |
|
- [**IMS Toucan**](https://github.com/DigitalPhonetics/IMS-Toucan), which supports 7000 languages. [^4] |
|
|
|
Voice conversion is currently achieved through Coqui. |
|
|
|
Notes: |
|
1. ESpeak-NG seems to have the worst performance out of the box, but it has a lot of options for controlling voice output. |
|
2. Where a synthesizer supports multiple models/voices, I manually pick the appropriate model. |
|
3. Not all synthesizers support a given language. |
|
|
|
|
|
|
|
[^1]: Endpoints used are of the form https://huggingface.co/facebook/mms-tts-[LANG]. |
|
Learn more: |
|
[Docs](https://huggingface.co/docs/transformers/model_doc/mms) | |
|
[Paper](https://arxiv.org/abs/2305.13516) | |
|
[Supported languages](https://dl.fbaipublicfiles.com/mms/misc/language_coverage_mms.html) |
|
|
|
[^2]: [Available models](https://github.com/coqui-ai/TTS/blob/dev/TTS/.models.json) |
|
[^3]: [Language list](https://github.com/espeak-ng/espeak-ng/blob/master/docs/languages.md) |
|
[^4]: Language list is available in the Gradio API documentation [here](https://huggingface.co/spaces/Flux9665/MassivelyMultilingualTTS). |
|
''') |
|
|
|
|
