Update app.py

app.py

@@ -8,26 +8,30 @@ from transformers import SpeechT5ForTextToSpeech, SpeechT5HifiGan, SpeechT5Proce
 device = "cuda:0" if torch.cuda.is_available() else "cpu"
 
 # load speech translation checkpoint
-asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-base", device=device)
+asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-large-v2", device=device)
 
 # load text-to-speech checkpoint and speaker embeddings
-model_id = "burraco135/speecht5_finetuned_voxpopuli_it"  # update with your model id
+model_id = "Sandiago21/speecht5_finetuned_voxpopuli_it"  # update with your model id
 # pipe = pipeline("automatic-speech-recognition", model=model_id)
 model = SpeechT5ForTextToSpeech.from_pretrained(model_id)
-processor = SpeechT5Processor.from_pretrained(model_id)
 vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
-embeddings_dataset = np.load("speaker_0_embeddings.npy")
-speaker_embeddings = torch.tensor(embeddings_dataset).unsqueeze(0)
+embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")
+speaker_embeddings = torch.tensor(embeddings_dataset[7440]["xvector"]).unsqueeze(0)
+
+processor = SpeechT5Processor.from_pretrained(model_id)
 
 replacements = [
-    ("à", "a"),
-    ("è", "e"),
-    ("ì", "i"),
-    ("í", "i"),
-    ("ï", "i"),
-    ("ò", "o"),
-    ("ó", "o"),
-    ("ù", "u")
+     ("á", "a"),
+     ("ç", "c"),
+     ("è", "e"),
+     ("ì", "i"),
+     ("í", "i"),
+     ("ò", "o"),
+     ("ó", "o"),
+     ("ù", "u"),
+     ("ú", "u"),
+     ("š", "s"),
+     ("ï", "i"),
 ]
 
 def cleanup_text(text):
@@ -35,13 +39,19 @@ def cleanup_text(text):
         text = text.replace(src, dst)
     return text
 
+def synthesize_speech(text):
+    text = cleanup_text(text)
+    inputs = processor(text=text, return_tensors="pt")
+    speech = model.generate_speech(inputs["input_ids"].to(device), speaker_embeddings.to(device), vocoder=vocoder)
+
+    return gr.Audio.update(value=(16000, speech.cpu().numpy()))
 
-def transcribe_to_german(audio):
+def translate(audio):
     outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"task": "transcribe", "language": "italian"})
     return outputs["text"]
 
 
-def synthesise_from_german(text):
+def synthesise(text):
     text = cleanup_text(text)
     inputs = processor(text=text, return_tensors="pt")
     speech = model.generate_speech(inputs["input_ids"].to(device), speaker_embeddings.to(device), vocoder=vocoder)
@@ -49,15 +59,15 @@ def synthesise_from_german(text):
 
 
 def speech_to_speech_translation(audio):
-    translated_text = transcribe_to_german(audio)
-    synthesised_speech = synthesise_from_german(translated_text)
+    translated_text = translate(audio)
+    synthesised_speech = synthesise(translated_text)
     synthesised_speech = (synthesised_speech.numpy() * 32767).astype(np.int16)
-    return ((16000, synthesised_speech), translated_text)
+    return 16000, synthesised_speech
 
 
 title = "Cascaded STST"
 description = """
-Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in German. Demo uses OpenAI's [Whisper Base](https://huggingface.co/openai/whisper-base) model for speech translation, and [burraco135/speecht5_finetuned_voxpopuli_it](https://huggingface.co/burraco135/speecht5_finetuned_voxpopuli_it) checkpoint for text-to-speech, which is based on Microsoft's
+Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in Italian. Demo uses OpenAI's [Whisper Large v2](https://huggingface.co/openai/whisper-large-v2) model for speech translation, and [Sandiago21/speecht5_finetuned_voxpopuli_it](https://huggingface.co/Sandiago21/speecht5_finetuned_voxpopuli_it) checkpoint for text-to-speech, which is based on Microsoft's
 [SpeechT5 TTS](https://huggingface.co/microsoft/speecht5_tts) model for text-to-speech, fine-tuned in Italian Audio dataset:
 ![Cascaded STST](https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/s2st_cascaded.png "Diagram of cascaded speech to speech translation")
 """
@@ -67,7 +77,7 @@ demo = gr.Blocks()
 mic_translate = gr.Interface(
     fn=speech_to_speech_translation,
     inputs=gr.Audio(source="microphone", type="filepath"),
-    outputs=[gr.Audio(label="Generated Speech", type="numpy"), gr.outputs.Textbox()],
+    outputs=gr.Audio(label="Generated Speech", type="numpy"),
     title=title,
     description=description,
 )
@@ -75,7 +85,7 @@ mic_translate = gr.Interface(
 file_translate = gr.Interface(
     fn=speech_to_speech_translation,
     inputs=gr.Audio(source="upload", type="filepath"),
-    outputs=[gr.Audio(label="Generated Speech", type="numpy"), gr.outputs.Textbox()],
+    outputs=gr.Audio(label="Generated Speech", type="numpy"),
     examples=[["./example.wav"]],
     title=title,
     description=description,
@@ -84,4 +94,4 @@ file_translate = gr.Interface(
 with demo:
     gr.TabbedInterface([mic_translate, file_translate], ["Microphone", "Audio File"])
 
-demo.launch()
+demo.launch()