import torch import scipy import os import streamlit as st import pandas as pd from transformers import pipeline #set_seed, from transformers import VitsTokenizer, VitsModel from datasets import load_dataset, Audio from huggingface_hub.inference_api import InferenceApi from src import * ######################## col1, col2 = st.columns([20,3]) with col2: st.image('logo.png', use_column_width=True) with col1: st.title("Mockingbird") st.header("A demo of open Text to Speech tools") tts, about = st.tabs(["Text to speech", "**About**"]) ######################## with tts: # Configurations -- language choice and text tts_lang = st.selectbox('Language of text', (language_list), format_func = decode_iso) tts_text = st.text_area(label = "Please enter your sentence here:", value="", placeholder=placeholders[tts_lang] ) target_speaker_file = st.file_uploader("If you would like to test voice conversion, you may upload your audio below. You should upload one file in .wav format. If you don't, a default file will be used.", type=['wav']) # Inference if st.button("Generate"): # Warning about alphabet support if tts_lang in ['rus', 'fas']: st.warning("WARNING! On Windows, ESpeak-NG has trouble synthesizing output when input is provided from non-Latin alphabets.") st.divider() # Synthesis with st.spinner(":rainbow[Synthesizing, please wait... (this will be slowest the first time you generate audio in a new language)]"): if tts_text == "": tts_text=placeholders[tts_lang] # First, make the audio base_mms = synth_mms(tts_text, models[tts_lang]['mms']) base_coqui= synth_coqui(tts_text, models[tts_lang]['coqui']) base_espeakng= synth_espeakng(tts_text, models[tts_lang]['espeakng']) base_toucan= synth_toucan(tts_text, models[tts_lang]['toucan']) base_piper synth_piper(tts_text, models[tts_lang]['piper']) if tts_lang=="swh": finetuned_mms1 = synth_mms(tts_text, "khof312/mms-tts-swh-female-1") finetuned_mms2 = synth_mms(tts_text, "khof312/mms-tts-swh-female-2") if tts_lang=="spa": finetuned_mms1 = synth_mms(tts_text, "ylacombe/mms-spa-finetuned-argentinian-monospeaker") finetuned_mms2 = synth_mms(tts_text, "ylacombe/mms-spa-finetuned-chilean-monospeaker") finetuned_mms3 = synth_mms(tts_text, "ylacombe/mms-spa-finetuned-colombian-monospeaker") finetuned_mms4 = synth_mms(tts_text, "khof312/mms-tts-spa-female") if tts_lang=="lin": finetuned_mms1 = synth_mms(tts_text, "khof312/mms-tts-lin-female") #vc_mms #vc_coqui #vc_espeakng "## Synthesis" "### Default models" row1 = st.columns([1,1,2]) row2 = st.columns([1,1,2]) row3 = st.columns([1,1,2]) row4 = st.columns([1,1,2]) row5 = st.columns([1,1,2]) row6 = st.columns([1,1,2]) row1[0].write("**Model**") row1[1].write("**Configuration**") row1[2].write("**Audio**") if base_mms is not None: row2[0].write(f"[Meta MMS](https://huggingface.co/docs/transformers/main/en/model_doc/mms)") row2[1].write("default") row2[2].audio(base_mms[0], sample_rate = base_mms[1]) if base_coqui is not None: row3[0].write(f"[Coqui](https://docs.coqui.ai/en/latest/index.html)") row3[1].write("default") row3[2].audio(base_coqui[0], sample_rate = base_coqui[1]) if base_espeakng is not None: row4[0].write(f"[Espeak-ng](https://github.com/espeak-ng/espeak-ng)") row4[1].write("default") row4[2].audio(base_espeakng[0], sample_rate = base_espeakng[1]) row5[0].write(f"[IMS-Toucan](https://github.com/DigitalPhonetics/IMS-Toucan)") row5[1].write("default") row5[2].audio(base_toucan[0], sample_rate = base_toucan[1]) if base_piper is not None: row6[0].write(f"[Piper](https://github.com/rhasspy/piper)") row6[1].write("default") row6[2].audio(base_piper[0], sample_rate = base_piper[1]) ################################################################# if tts_lang == "swh": "### Fine Tuned" row1 = st.columns([1,1,2]) row2 = st.columns([1,1,2]) row3 = st.columns([1,1,2]) row1[0].write("**Model**") row1[1].write("**Configuration**") row1[2].write("**Audio**") row2[0].write(f"Meta MMS") row2[1].write("[female 1](https://huggingface.co/khof312/mms-tts-swh-female-1)") row2[2].audio(finetuned_mms1[0], sample_rate = finetuned_mms1[1]) row3[0].write(f"Meta MMS") row3[1].write("[female 2](https://huggingface.co/khof312/mms-tts-swh-female-2)") row3[2].audio(finetuned_mms2[0], sample_rate = finetuned_mms2[1]) if tts_lang == "spa": "### Fine Tuned" row1 = st.columns([1,1,2]) row2 = st.columns([1,1,2]) row3 = st.columns([1,1,2]) row4 = st.columns([1,1,2]) row5 = st.columns([1,1,2]) row1[0].write("**Model**") row1[1].write("**Configuration**") row1[2].write("**Audio**") row2[0].write(f"Meta MMS") row2[1].write("[ylacombe - Argentinian](https://huggingface.co/ylacombe/mms-spa-finetuned-argentinian-monospeaker)") row2[2].audio(finetuned_mms1[0], sample_rate = finetuned_mms1[1]) row3[0].write(f"Meta MMS") row3[1].write("[ylacombe - Chilean](https://huggingface.co/ylacombe/mms-spa-finetuned-chilean-monospeaker)") row3[2].audio(finetuned_mms2[0], sample_rate = finetuned_mms2[1]) row4[0].write(f"Meta MMS") row4[1].write("[ylacombe - Colombian](https://huggingface.co/ylacombe/mms-spa-finetuned-colombian-monospeaker)") row4[2].audio(finetuned_mms3[0], sample_rate = finetuned_mms3[1]) row5[0].write(f"Meta MMS") row5[1].write("[khof312 - female](https://huggingface.co/khof312/mms-tts-spa-female)") row5[2].audio(finetuned_mms4[0], sample_rate = finetuned_mms4[1]) if tts_lang == "lin": "### Fine Tuned" row1 = st.columns([1,1,2]) row2 = st.columns([1,1,2]) row1[0].write("**Model**") row1[1].write("**Configuration**") row1[2].write("**Audio**") row2[0].write(f"Meta MMS") row2[1].write("[khof312 - femalehttps://huggingface.co/khof312/mms-tts-lin-female)") row2[2].audio(finetuned_mms1[0], sample_rate = finetuned_mms1[1]) st.divider() "## Voice conversion" ################################################################# st.warning('''Note: The naturalness of the audio will only be as good as that of the audio in "default models" above.''') if target_speaker_file is not None: rate, wav = scipy.io.wavfile.read(target_speaker_file) scipy.io.wavfile.write("target_speaker_custom.wav", data=wav, rate=rate) target_speaker = "target_speaker_custom.wav" else: target_speaker = "target_speaker.wav" if base_mms is not None: scipy.io.wavfile.write("source_speaker_mms.wav", rate=base_mms[1], data=base_mms[0].T) converted_mms = convert_coqui('source_speaker_mms.wav', target_speaker) if base_coqui is not None: scipy.io.wavfile.write("source_speaker_coqui.wav", rate=base_coqui[1], data=base_coqui[0].T) converted_coqui = convert_coqui('source_speaker_coqui.wav', target_speaker) if base_espeakng is not None: scipy.io.wavfile.write("source_speaker_espeakng.wav", rate=base_espeakng[1], data=base_espeakng[0].T) converted_espeakng = convert_coqui('source_speaker_espeakng.wav', target_speaker) scipy.io.wavfile.write("source_speaker_toucan.wav", rate=base_toucan[1], data=base_toucan[0].T) converted_toucan = convert_coqui('source_speaker_toucan.wav', target_speaker) row1 = st.columns([1,1,2]) row2 = st.columns([1,1,2]) row3 = st.columns([1,1,2]) row4 = st.columns([1,1,2]) row1[0].write("**Model**") row1[1].write("**Configuration**") row1[2].write("**Audio**") if base_mms is not None: row1[0].write(f"Meta MMS") row1[1].write(f"converted") row1[2].audio(converted_mms[0], sample_rate = converted_mms[1]) if base_coqui is not None: row2[0].write(f"Coqui") row2[1].write(f"converted") row2[2].audio(converted_coqui[0], sample_rate = converted_coqui[1]) if base_espeakng is not None: row3[0].write(f"Espeak-ng") row3[1].write(f"converted") row3[2].audio(converted_espeakng[0], sample_rate = converted_espeakng[1]) row4[0].write(f"IMS Toucan") row4[1].write(f"converted") row4[2].audio(converted_toucan[0], sample_rate = converted_toucan[1]) #row3[0].write("MMS-TTS-SWH") #row3[1].audio(synth, sample_rate=16_000) #row3[2].audio(synth, sample_rate=16_000) #st.audio(synth, sample_rate=16_000) #data.write(np.random.randn(10, 1) #col1.subheader("A wide column with a chart") #col1.line_chart(data) #col2.subheader("A narrow column with the data") #col2.write(data) with about: #st.header("How it works") st.markdown('''# Mockingbird TTS Demo This page is a demo of the openly available Text to Speech models for various languages of interest. Currently, 3 synthesizers with multilingual offerings are supported out of the box: - [**Meta's Massively Multilingual Speech (MMS)**](https://ai.meta.com/blog/multilingual-model-speech-recognition/) model, which supports over 1000 languages.[^1] - [**IMS Toucan**](https://github.com/DigitalPhonetics/IMS-Toucan), which supports 7000 languages.[^4] - [**ESpeak-NG's**](https://github.com/espeak-ng/espeak-ng/tree/master)'s synthetic voices**[^3] On a case-by-case basis, for different languages of interest, I have added: - [**Coqui's TTS**](https://docs.coqui.ai/en/latest/#) package;[^2] while no longer supported, Coqui acted as a hub for TTS model hosting and these models are still available. Languages must be added on a model-by-model basis. - Specific fine-tuned variants of Meta's MMS (either fine-tuned by [Yoach Lacombe](https://huggingface.co/ylacombe), or fine-tuned by me using his scripts). I am in the process of adding support for: - [**Piper**](https://github.com/rhasspy/piper), a TTS system that supports multiple voices per language and approximately 30 languages. To test different voices, please see the [Huggingface demo](https://huggingface.co/spaces/k2-fsa/text-to-speech).[^5] - [**African Voices**](https://github.com/neulab/AfricanVoices), a CMU research project that fine-tuned synthesizers for different African languages. The site hosting the synthesizers is deprecated but they can be downloaded from Google's Wayback Machine. [^6] Voice conversion is currently achieved through Coqui. Notes: 1. ESpeak-NG seems to have the worst performance out of the box, but it has a lot of options for controlling voice output. 2. Where a synthesizer supports multiple models/voices, I manually pick the appropriate model. 3. Not all synthesizers support a given language. [^1]: Endpoints used are of the form https://huggingface.co/facebook/mms-tts-[LANG]. Learn more: [Docs](https://huggingface.co/docs/transformers/model_doc/mms) | [Paper](https://arxiv.org/abs/2305.13516) | [Supported languages](https://dl.fbaipublicfiles.com/mms/misc/language_coverage_mms.html) [^2]: [Available models](https://github.com/coqui-ai/TTS/blob/dev/TTS/.models.json) [^3]: [Language list](https://github.com/espeak-ng/espeak-ng/blob/master/docs/languages.md) [^4]: Language list is available in the Gradio API documentation [here](https://huggingface.co/spaces/Flux9665/MassivelyMultilingualTTS). [^5]: The list of available voices is [here](https://github.com/rhasspy/piper/blob/master/VOICES.md), model checkpoints are [here](https://huggingface.co/datasets/rhasspy/piper-checkpoints/tree/main), and they can be tested [here](https://rhasspy.github.io/piper-samples/). [^6]: ''')