File size: 3,136 Bytes
d347764 4939ea5 d347764 ba496cd 4939ea5 d347764 4939ea5 d347764 ba496cd d347764 4939ea5 ba496cd d347764 4939ea5 d347764 ba496cd d347764 f805e49 4939ea5 f805e49 8c5de0d c737803 d347764 4939ea5 d347764 f805e49 d347764 c737803 4939ea5 c737803 974405a 8c5de0d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 |
import gradio as gr
import numpy as np
import torch
from datasets import load_dataset
from transformers import SpeechT5ForTextToSpeech, SpeechT5HifiGan, SpeechT5Processor, pipeline
device = "cuda:0" if torch.cuda.is_available() else "cpu"
# load speech translation checkpoint
asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-base", device=device)
# load text-to-speech checkpoint and speaker embeddings
tts_checkpoint = "sanchit-gandhi/speecht5_tts_vox_nl"
processor = SpeechT5Processor.from_pretrained(tts_checkpoint)
model = SpeechT5ForTextToSpeech.from_pretrained(tts_checkpoint).to(device)
vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan").to(device)
embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")
speaker_embeddings = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0)
def translate(audio):
# Trick Whisper to translate from any language to Dutch.
# Note that using task=translate will translate to English instead.
outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"task": "transcribe", "language": "dutch"})
return outputs["text"]
def synthesise(text):
# Need to specific truncate to max text positions.
# Otherwise model.generate_speech will throw errors.
inputs = processor(
text=text,
# max_length=200,
max_length=598,
truncation=True,
# padding=True,
return_tensors="pt"
)
# inputs = processor(text=text, return_tensors="pt")
speech = model.generate_speech(inputs["input_ids"].to(device), speaker_embeddings.to(device), vocoder=vocoder)
return speech.cpu()
def speech_to_speech_translation(audio):
translated_text = translate(audio)
synthesised_speech = synthesise(translated_text)
synthesised_speech = (synthesised_speech.numpy() * 32767).astype(np.int16)
return 16000, synthesised_speech
title = "Cascaded STST"
description = """
Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in Dutch. Demo uses OpenAI's [Whisper Tiny](https://huggingface.co/openai/whisper-tiny) model for speech translation, and a finetuned
[SpeechT5 TTS](https://huggingface.co/sanchit-gandhi/speecht5_tts_vox_nl) model for text-to-speech:
"""
demo = gr.Blocks()
mic_translate = gr.Interface(
fn=speech_to_speech_translation,
inputs=gr.Audio(label="microphone", type="filepath"),
outputs=gr.Audio(label="Generated Speech", type="numpy"),
title=title,
description=description,
)
file_translate = gr.Interface(
fn=speech_to_speech_translation,
inputs=gr.Audio(label="upload", type="filepath"),
outputs=gr.Audio(label="Generated Speech", type="numpy"),
examples=[["./example.wav"]],
title=title,
description=description,
)
with demo:
gr.TabbedInterface([mic_translate, file_translate], ["Microphone", "Audio File"])
demo.launch() |