Spaces:
Running
Running
| import torch | |
| import scipy | |
| import os | |
| import streamlit as st | |
| import pandas as pd | |
| from transformers import pipeline #set_seed, | |
| from transformers import VitsTokenizer, VitsModel | |
| from datasets import load_dataset, Audio | |
| from huggingface_hub.inference_api import InferenceApi | |
| from src import * | |
| ######################## | |
| col1, col2 = st.columns([20,3]) | |
| with col2: | |
| st.image('logo.png', use_column_width=True) | |
| with col1: | |
| st.title("Mockingbird") | |
| st.header("A demo of open Text to Speech tools") | |
| tts, about = st.tabs(["Text to speech", "**About**"]) | |
| ######################## | |
| with tts: | |
| # Configurations -- language choice and text | |
| tts_lang = st.selectbox('Language of text', (language_list), format_func = decode_iso) | |
| tts_text = st.text_area(label = "Please enter your sentence here:", | |
| value="", placeholder=placeholders[tts_lang] ) | |
| target_speaker_file = st.file_uploader("If you would like to test voice conversion, you may upload your audio below. You should upload one file in .wav format. If you don't, a default file will be used.", | |
| type=['wav']) | |
| # Inference | |
| if st.button("Generate"): | |
| # Warning about alphabet support | |
| if tts_lang in ['rus', 'fas']: | |
| st.warning("WARNING! On Windows, ESpeak-NG has trouble synthesizing output when input is provided from non-Latin alphabets.") | |
| st.divider() | |
| # Synthesis | |
| with st.spinner(":rainbow[Synthesizing, please wait... (this will be slowest the first time you generate audio in a new language)]"): | |
| if tts_text == "": | |
| tts_text=placeholders[tts_lang] | |
| # First, make the audio | |
| base_mms = synth_mms(tts_text, models[tts_lang]['mms']) | |
| base_coqui= synth_coqui(tts_text, models[tts_lang]['coqui']) | |
| base_espeakng= synth_espeakng(tts_text, models[tts_lang]['espeakng']) | |
| base_toucan= synth_toucan(tts_text, models[tts_lang]['toucan']) | |
| if tts_lang=="swh": | |
| finetuned_mms1 = synth_mms(tts_text, "khof312/mms-tts-swh-female-1") | |
| finetuned_mms2 = synth_mms(tts_text, "khof312/mms-tts-swh-female-2") | |
| if tts_lang=="spa": | |
| finetuned_mms1 = synth_mms(tts_text, "ylacombe/mms-spa-finetuned-argentinian-monospeaker") | |
| finetuned_mms2 = synth_mms(tts_text, "ylacombe/mms-spa-finetuned-chilean-monospeaker") | |
| finetuned_mms3 = synth_mms(tts_text, "ylacombe/mms-spa-finetuned-colombian-monospeaker") | |
| #vc_mms | |
| #vc_coqui | |
| #vc_espeakng | |
| "## Synthesis" | |
| "### Default models" | |
| row1 = st.columns([1,1,2]) | |
| row2 = st.columns([1,1,2]) | |
| row3 = st.columns([1,1,2]) | |
| row4 = st.columns([1,1,2]) | |
| row5 = st.columns([1,1,2]) | |
| row1[0].write("**Model**") | |
| row1[1].write("**Configuration**") | |
| row1[2].write("**Audio**") | |
| if base_mms is not None: | |
| row2[0].write(f"[Meta MMS](https://huggingface.co/docs/transformers/main/en/model_doc/mms)") | |
| row2[1].write("default") | |
| row2[2].audio(base_mms[0], sample_rate = base_mms[1]) | |
| if base_coqui is not None: | |
| row3[0].write(f"[Coqui](https://docs.coqui.ai/en/latest/index.html)") | |
| row3[1].write("default") | |
| row3[2].audio(base_coqui[0], sample_rate = base_coqui[1]) | |
| if base_espeakng is not None: | |
| row4[0].write(f"[Espeak-ng](https://github.com/espeak-ng/espeak-ng)") | |
| row4[1].write("default") | |
| row4[2].audio(base_espeakng[0], sample_rate = base_espeakng[1]) | |
| row5[0].write(f"[IMS-Toucan](https://github.com/DigitalPhonetics/IMS-Toucan)") | |
| row5[1].write("default") | |
| row5[2].audio(base_toucan[0], sample_rate = base_toucan[1]) | |
| ################################################################# | |
| if tts_lang == "swh": | |
| "### Fine Tuned" | |
| row1 = st.columns([1,1,2]) | |
| row2 = st.columns([1,1,2]) | |
| row3 = st.columns([1,1,2]) | |
| row1[0].write("**Model**") | |
| row1[1].write("**Configuration**") | |
| row1[2].write("**Audio**") | |
| row2[0].write(f"Meta MMS") | |
| row2[1].write("[female 1](https://huggingface.co/khof312/mms-tts-swh-female-1)") | |
| row2[2].audio(finetuned_mms1[0], sample_rate = finetuned_mms1[1]) | |
| row3[0].write(f"Meta MMS") | |
| row3[1].write("[female 2](https://huggingface.co/khof312/mms-tts-swh-female-2)") | |
| row3[2].audio(finetuned_mms2[0], sample_rate = finetuned_mms2[1]) | |
| if tts_lang == "spa": | |
| "### Fine Tuned" | |
| row1 = st.columns([1,1,2]) | |
| row2 = st.columns([1,1,2]) | |
| row3 = st.columns([1,1,2]) | |
| row4 = st.columns([1,1,2]) | |
| row1[0].write("**Model**") | |
| row1[1].write("**Configuration**") | |
| row1[2].write("**Audio**") | |
| row2[0].write(f"Meta MMS") | |
| row2[1].write("[ylacombe - Argentinian](https://huggingface.co/ylacombe/mms-spa-finetuned-argentinian-monospeaker)") | |
| row2[2].audio(finetuned_mms1[0], sample_rate = finetuned_mms1[1]) | |
| row3[0].write(f"Meta MMS") | |
| row3[1].write("[ylacombe - Chilean](https://huggingface.co/ylacombe/mms-spa-finetuned-chilean-monospeaker)") | |
| row3[2].audio(finetuned_mms2[0], sample_rate = finetuned_mms2[1]) | |
| row4[0].write(f"Meta MMS") | |
| row4[1].write("[ylacombe - Colombian](https://huggingface.co/ylacombe/mms-spa-finetuned-colombian-monospeaker)") | |
| row4[2].audio(finetuned_mms3[0], sample_rate = finetuned_mms3[1]) | |
| st.divider() | |
| "## Voice conversion" ################################################################# | |
| st.warning('''Note: The naturalness of the audio will only be as good as that of the audio in "default models" above.''') | |
| if target_speaker_file is not None: | |
| rate, wav = scipy.io.wavfile.read(target_speaker_file) | |
| scipy.io.wavfile.write("target_speaker_custom.wav", data=wav, rate=rate) | |
| target_speaker = "target_speaker_custom.wav" | |
| else: | |
| target_speaker = "target_speaker.wav" | |
| if base_mms is not None: | |
| scipy.io.wavfile.write("source_speaker_mms.wav", rate=base_mms[1], data=base_mms[0].T) | |
| converted_mms = convert_coqui('source_speaker_mms.wav', target_speaker) | |
| if base_coqui is not None: | |
| scipy.io.wavfile.write("source_speaker_coqui.wav", rate=base_coqui[1], data=base_coqui[0].T) | |
| converted_coqui = convert_coqui('source_speaker_coqui.wav', target_speaker) | |
| if base_espeakng is not None: | |
| scipy.io.wavfile.write("source_speaker_espeakng.wav", rate=base_espeakng[1], data=base_espeakng[0].T) | |
| converted_espeakng = convert_coqui('source_speaker_espeakng.wav', target_speaker) | |
| scipy.io.wavfile.write("source_speaker_toucan.wav", rate=base_toucan[1], data=base_toucan[0].T) | |
| converted_toucan = convert_coqui('source_speaker_toucan.wav', target_speaker) | |
| row1 = st.columns([1,1,2]) | |
| row2 = st.columns([1,1,2]) | |
| row3 = st.columns([1,1,2]) | |
| row4 = st.columns([1,1,2]) | |
| row1[0].write("**Model**") | |
| row1[1].write("**Configuration**") | |
| row1[2].write("**Audio**") | |
| if base_mms is not None: | |
| row1[0].write(f"Meta MMS") | |
| row1[1].write(f"converted") | |
| row1[2].audio(converted_mms[0], sample_rate = converted_mms[1]) | |
| if base_coqui is not None: | |
| row2[0].write(f"Coqui") | |
| row2[1].write(f"converted") | |
| row2[2].audio(converted_coqui[0], sample_rate = converted_coqui[1]) | |
| if base_espeakng is not None: | |
| row3[0].write(f"Espeak-ng") | |
| row3[1].write(f"converted") | |
| row3[2].audio(converted_espeakng[0], sample_rate = converted_espeakng[1]) | |
| row4[0].write(f"IMS Toucan") | |
| row4[1].write(f"converted") | |
| row4[2].audio(converted_toucan[0], sample_rate = converted_toucan[1]) | |
| #row3[0].write("MMS-TTS-SWH") | |
| #row3[1].audio(synth, sample_rate=16_000) | |
| #row3[2].audio(synth, sample_rate=16_000) | |
| #st.audio(synth, sample_rate=16_000) | |
| #data.write(np.random.randn(10, 1) | |
| #col1.subheader("A wide column with a chart") | |
| #col1.line_chart(data) | |
| #col2.subheader("A narrow column with the data") | |
| #col2.write(data) | |
| with about: | |
| #st.header("How it works") | |
| st.markdown('''# Mockingbird TTS Demo | |
| This page is a demo of the openly available Text to Speech models for various languages of interest. Currently, 4 synthesizers are supported: | |
| - [**Meta's Massively Multilingual Speech (MMS)**](https://ai.meta.com/blog/multilingual-model-speech-recognition/) model, which supports over 1000 languages.[^1] | |
| - [**Coqui's TTS**](https://docs.coqui.ai/en/latest/#) package;[^2] while no longer supported, Coqui acted as a hub for TTS model hosting and these models are still available. | |
| - [**ESpeak-NG's**](https://github.com/espeak-ng/espeak-ng/tree/master)'s synthetic voices**[^3] | |
| - [**IMS Toucan**](https://github.com/DigitalPhonetics/IMS-Toucan), which supports 7000 languages. [^4] | |
| Voice conversion is currently achieved through Coqui. | |
| Notes: | |
| 1. ESpeak-NG seems to have the worst performance out of the box, but it has a lot of options for controlling voice output. | |
| 2. Where a synthesizer supports multiple models/voices, I manually pick the appropriate model. | |
| 3. Not all synthesizers support a given language. | |
| [^1]: Endpoints used are of the form https://huggingface.co/facebook/mms-tts-[LANG]. | |
| Learn more: | |
| [Docs](https://huggingface.co/docs/transformers/model_doc/mms) | | |
| [Paper](https://arxiv.org/abs/2305.13516) | | |
| [Supported languages](https://dl.fbaipublicfiles.com/mms/misc/language_coverage_mms.html) | |
| [^2]: [Available models](https://github.com/coqui-ai/TTS/blob/dev/TTS/.models.json) | |
| [^3]: [Language list](https://github.com/espeak-ng/espeak-ng/blob/master/docs/languages.md) | |
| [^4]: Language list is available in the Gradio API documentation [here](https://huggingface.co/spaces/Flux9665/MassivelyMultilingualTTS). | |
| ''') | |