tsobolev commited on
Commit
9ac287c
1 Parent(s): c1ed890

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +54 -64
app.py CHANGED
@@ -21,67 +21,57 @@ embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validat
21
  speaker_embeddings = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0)
22
 
23
 
24
- print(speaker_embeddings.shape)
25
-
26
-
27
- if 0:
28
-
29
- def translate(audio):
30
- outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"task": "translate"})
31
- return outputs["text"]
32
-
33
-
34
- def synthesise(text):
35
- inputs = processor(text=text, return_tensors="pt")
36
- speech = model.generate_speech(inputs["input_ids"].to(device), speaker_embeddings.to(device), vocoder=vocoder)
37
- return speech.cpu()
38
-
39
-
40
- def speech_to_speech_translation(audio):
41
- translated_text = translate(audio)
42
- synthesised_speech = synthesise(translated_text)
43
- synthesised_speech = (synthesised_speech.numpy() * 32767).astype(np.int16)
44
- return 16000, synthesised_speech
45
-
46
-
47
- title = "Cascaded STST"
48
- description = """
49
- Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in English. Demo uses OpenAI's [Whisper Base](https://huggingface.co/openai/whisper-base) model for speech translation, and Microsoft's
50
- [SpeechT5 TTS](https://huggingface.co/microsoft/speecht5_tts) model for text-to-speech:
51
-
52
- ![Cascaded STST](https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/s2st_cascaded.png "Diagram of cascaded speech to speech translation")
53
- """
54
-
55
- demo = gr.Blocks()
56
-
57
- mic_translate = gr.Interface(
58
- fn=speech_to_speech_translation,
59
- inputs=gr.Audio(source="microphone", type="filepath"),
60
- outputs=gr.Audio(label="Generated Speech", type="numpy"),
61
- title=title,
62
- description=description,
63
- )
64
-
65
- file_translate = gr.Interface(
66
- fn=speech_to_speech_translation,
67
- inputs=gr.Audio(source="upload", type="filepath"),
68
- outputs=gr.Audio(label="Generated Speech", type="numpy"),
69
- examples=[["./example.wav"]],
70
- title=title,
71
- description=description,
72
- )
73
-
74
- with demo:
75
- gr.TabbedInterface([mic_translate, file_translate], ["Microphone", "Audio File"])
76
-
77
- demo.launch()
78
-
79
- else:
80
-
81
- def greet(name):
82
- return "Hello " + name
83
-
84
-
85
- demo = gr.Interface(fn=greet, inputs="text", outputs="text")
86
-
87
- demo.launch()
 
21
  speaker_embeddings = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0)
22
 
23
 
24
+ print("Blocks interface does not work, hmm",gr.__version__)
25
+
26
+
27
+
28
+
29
+ def translate(audio):
30
+ outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"task": "translate"})
31
+ return outputs["text"]
32
+
33
+
34
+ def synthesise(text):
35
+ inputs = processor(text=text, return_tensors="pt")
36
+ speech = model.generate_speech(inputs["input_ids"].to(device), speaker_embeddings.to(device), vocoder=vocoder)
37
+ return speech.cpu()
38
+
39
+
40
+ def speech_to_speech_translation(audio):
41
+ translated_text = translate(audio)
42
+ synthesised_speech = synthesise(translated_text)
43
+ synthesised_speech = (synthesised_speech.numpy() * 32767).astype(np.int16)
44
+ return 16000, synthesised_speech
45
+
46
+
47
+ title = "Cascaded STST"
48
+ description = """
49
+ Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in English. Demo uses OpenAI's [Whisper Base](https://huggingface.co/openai/whisper-base) model for speech translation, and Microsoft's
50
+ [SpeechT5 TTS](https://huggingface.co/microsoft/speecht5_tts) model for text-to-speech:
51
+
52
+ ![Cascaded STST](https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/s2st_cascaded.png "Diagram of cascaded speech to speech translation")
53
+ """
54
+
55
+ demo = gr.Blocks()
56
+
57
+ mic_translate = gr.Interface(
58
+ fn=speech_to_speech_translation,
59
+ inputs=gr.Audio(source="microphone", type="filepath"),
60
+ outputs=gr.Audio(label="Generated Speech", type="numpy"),
61
+ title=title,
62
+ description=description,
63
+ )
64
+
65
+ file_translate = gr.Interface(
66
+ fn=speech_to_speech_translation,
67
+ inputs=gr.Audio(source="upload", type="filepath"),
68
+ outputs=gr.Audio(label="Generated Speech", type="numpy"),
69
+ examples=[["./example.wav"]],
70
+ title=title,
71
+ description=description,
72
+ )
73
+
74
+ with demo:
75
+ gr.TabbedInterface([mic_translate, file_translate], ["Microphone", "Audio File"])
76
+
77
+ demo.launch()