fx dir

app.py

@@ -0,0 +1,187 @@
+INTROTXT = """# StyleTTS 2
+
+[Paper](https://arxiv.org/abs/2306.07691) - [Samples](https://styletts2.github.io/) - [Code](https://github.com/yl4579/StyleTTS2) - [Discord](https://discord.gg/ha8sxdG2K4)
+
+A free demo of StyleTTS 2. **I am not affiliated with the StyleTTS 2 Authors.**
+
+**Before using this demo, you agree to inform the listeners that the speech samples are synthesized by the pre-trained models, unless you have the permission to use the voice you synthesize. That is, you agree to only use voices whose speakers grant the permission to have their voice cloned, either directly or by license before making synthesized voices public, or you have to publicly announce that these voices are synthesized if you do not have the permission to use these voices.**
+
+Is there a long queue on this space? Duplicate it and add a more powerful GPU to skip the wait! **Note: Thank you to Hugging Face for their generous GPU grant program!**
+
+**NOTE: StyleTTS 2 does better on longer texts.** For example, making it say "hi" will produce a lower-quality result than making it say a longer phrase.
+
+**NOTE: StyleTTS 2 is _currently_ English-only. Join the Discord for updates on multilingual training.**
+"""
+import gradio as gr
+import styletts2importable
+import ljspeechimportable
+import torch
+import os
+from txtsplit import txtsplit
+import numpy as np
+import pickle
+theme = gr.themes.Base(
+    font=[gr.themes.GoogleFont('Libre Franklin'), gr.themes.GoogleFont('Public Sans'), 'system-ui', 'sans-serif'],
+)
+voicelist = ['f-us-1', 'f-us-2', 'f-us-3', 'f-us-4', 'm-us-1', 'm-us-2', 'm-us-3', 'm-us-4']
+voices = {}
+import phonemizer
+global_phonemizer = phonemizer.backend.EspeakBackend(language='en-us', preserve_punctuation=True,  with_stress=True)
+# todo: cache computed style, load using pickle
+# if os.path.exists('voices.pkl'):
+    # with open('voices.pkl', 'rb') as f:
+        # voices = pickle.load(f)
+# else:
+for v in voicelist:
+    voices[v] = styletts2importable.compute_style(f'voices/{v}.wav')
+# def synthesize(text, voice, multispeakersteps):
+#     if text.strip() == "":
+#         raise gr.Error("You must enter some text")
+#     # if len(global_phonemizer.phonemize([text])) > 300:
+#     if len(text) > 300:
+#         raise gr.Error("Text must be under 300 characters")
+#     v = voice.lower()
+#     # return (24000, styletts2importable.inference(text, voices[v], alpha=0.3, beta=0.7, diffusion_steps=7, embedding_scale=1))
+#     return (24000, styletts2importable.inference(text, voices[v], alpha=0.3, beta=0.7, diffusion_steps=multispeakersteps, embedding_scale=1))
+if not torch.cuda.is_available(): INTROTXT += "\n\n### You are on a CPU-only system, inference will be much slower.\n\nYou can use the [online demo](https://huggingface.co/spaces/styletts2/styletts2) for fast inference."
+def synthesize(text, voice, lngsteps, password, progress=gr.Progress()):
+    if text.strip() == "":
+        raise gr.Error("You must enter some text")
+    if len(text) > 50000:
+        raise gr.Error("Text must be <50k characters")
+    print("*** saying ***")
+    print(text)
+    print("*** end ***")
+    texts = txtsplit(text)
+    v = voice.lower()
+    audios = []
+    for t in progress.tqdm(texts):
+        print(t)
+        audios.append(styletts2importable.inference(t, voices[v], alpha=0.3, beta=0.7, diffusion_steps=lngsteps, embedding_scale=1))
+    return (24000, np.concatenate(audios))
+# def longsynthesize(text, voice, lngsteps, password, progress=gr.Progress()):
+#     if password == os.environ['ACCESS_CODE']:
+#         if text.strip() == "":
+#             raise gr.Error("You must enter some text")
+#         if lngsteps > 25:
+#             raise gr.Error("Max 25 steps")
+#         if lngsteps < 5:
+#             raise gr.Error("Min 5 steps")
+#         texts = split_and_recombine_text(text)
+#         v = voice.lower()
+#         audios = []
+#         for t in progress.tqdm(texts):
+#             audios.append(styletts2importable.inference(t, voices[v], alpha=0.3, beta=0.7, diffusion_steps=lngsteps, embedding_scale=1))
+#         return (24000, np.concatenate(audios))
+#     else:
+#         raise gr.Error('Wrong access code')
+def clsynthesize(text, voice, vcsteps, embscale, alpha, beta, progress=gr.Progress()):
+    # if text.strip() == "":
+    #     raise gr.Error("You must enter some text")
+    # # if global_phonemizer.phonemize([text]) > 300:
+    # if len(text) > 400:
+    #     raise gr.Error("Text must be under 400 characters")
+    # # return (24000, styletts2importable.inference(text, styletts2importable.compute_style(voice), alpha=0.3, beta=0.7, diffusion_steps=20, embedding_scale=1))
+    # return (24000, styletts2importable.inference(text, styletts2importable.compute_style(voice), alpha=0.3, beta=0.7, diffusion_steps=vcsteps, embedding_scale=1))
+    if text.strip() == "":
+        raise gr.Error("You must enter some text")
+    if len(text) > 50000:
+        raise gr.Error("Text must be <50k characters")
+    if embscale > 1.3 and len(text) < 20:
+        gr.Warning("WARNING: You entered short text, you may get static!")
+    print("*** saying ***")
+    print(text)
+    print("*** end ***")
+    texts = txtsplit(text)
+    audios = []
+    # vs = styletts2importable.compute_style(voice)
+    vs = styletts2importable.compute_style(voice)
+    # print(vs)
+    for t in progress.tqdm(texts):
+        audios.append(styletts2importable.inference(t, vs, alpha=alpha, beta=beta, diffusion_steps=vcsteps, embedding_scale=embscale))
+        # audios.append(styletts2importable.inference(t, vs, diffusion_steps=10, alpha=0.3, beta=0.7, embedding_scale=5))
+    return (24000, np.concatenate(audios))
+def ljsynthesize(text, steps, progress=gr.Progress()):
+    # if text.strip() == "":
+    #     raise gr.Error("You must enter some text")
+    # # if global_phonemizer.phonemize([text]) > 300:
+    # if len(text) > 400:
+    #     raise gr.Error("Text must be under 400 characters")
+    noise = torch.randn(1,1,256).to('cuda' if torch.cuda.is_available() else 'cpu')
+    # return (24000, ljspeechimportable.inference(text, noise, diffusion_steps=7, embedding_scale=1))
+    if text.strip() == "":
+        raise gr.Error("You must enter some text")
+    if len(text) > 150000:
+        raise gr.Error("Text must be <150k characters")
+    print("*** saying ***")
+    print(text)
+    print("*** end ***")
+    texts = txtsplit(text)
+    audios = []
+    for t in progress.tqdm(texts):
+        audios.append(ljspeechimportable.inference(t, noise, diffusion_steps=steps, embedding_scale=1))
+    return (24000, np.concatenate(audios))
+
+
+with gr.Blocks() as vctk:
+    with gr.Row():
+        with gr.Column(scale=1):
+            inp = gr.Textbox(label="Text", info="What would you like StyleTTS 2 to read? It works better on full sentences.", interactive=True)
+            voice = gr.Dropdown(voicelist, label="Voice", info="Select a default voice.", value='m-us-2', interactive=True)
+            multispeakersteps = gr.Slider(minimum=3, maximum=15, value=3, step=1, label="Diffusion Steps", info="Theoretically, higher should be better quality but slower, but we cannot notice a difference. Try with lower steps first - it is faster", interactive=True)
+            # use_gruut = gr.Checkbox(label="Use alternate phonemizer (Gruut) - Experimental")
+        with gr.Column(scale=1):
+            btn = gr.Button("Synthesize", variant="primary")
+            audio = gr.Audio(interactive=False, label="Synthesized Audio", waveform_options={'waveform_progress_color': '#3C82F6'})
+            btn.click(synthesize, inputs=[inp, voice, multispeakersteps], outputs=[audio], concurrency_limit=4)
+with gr.Blocks() as clone:
+    with gr.Row():
+        with gr.Column(scale=1):
+            clinp = gr.Textbox(label="Text", info="What would you like StyleTTS 2 to read? It works better on full sentences.", interactive=True)
+            clvoice = gr.Audio(label="Voice", interactive=True, type='filepath', max_length=300, waveform_options={'waveform_progress_color': '#3C82F6'})
+            vcsteps = gr.Slider(minimum=3, maximum=20, value=20, step=1, label="Diffusion Steps", info="Theoretically, higher should be better quality but slower, but we cannot notice a difference. Try with lower steps first - it is faster", interactive=True)
+            embscale = gr.Slider(minimum=1, maximum=10, value=1, step=0.1, label="Embedding Scale (READ WARNING BELOW)", info="Defaults to 1. WARNING: If you set this too high and generate text that's too short you will get static!", interactive=True)
+            alpha = gr.Slider(minimum=0, maximum=1, value=0.3, step=0.1, label="Alpha", info="Defaults to 0.3", interactive=True)
+            beta = gr.Slider(minimum=0, maximum=1, value=0.7, step=0.1, label="Beta", info="Defaults to 0.7", interactive=True)
+        with gr.Column(scale=1):
+            clbtn = gr.Button("Synthesize", variant="primary")
+            claudio = gr.Audio(interactive=False, label="Synthesized Audio", waveform_options={'waveform_progress_color': '#3C82F6'})
+            clbtn.click(clsynthesize, inputs=[clinp, clvoice, vcsteps, embscale, alpha, beta], outputs=[claudio], concurrency_limit=4)
+# with gr.Blocks() as longText:
+#     with gr.Row():
+#         with gr.Column(scale=1):
+#             lnginp = gr.Textbox(label="Text", info="What would you like StyleTTS 2 to read? It works better on full sentences.", interactive=True)
+#             lngvoice = gr.Dropdown(voicelist, label="Voice", info="Select a default voice.", value='m-us-1', interactive=True)
+#             lngsteps = gr.Slider(minimum=5, maximum=25, value=10, step=1, label="Diffusion Steps", info="Higher = better quality, but slower", interactive=True)
+#             lngpwd = gr.Textbox(label="Access code", info="This feature is in beta. You need an access code to use it as it uses more resources and we would like to prevent abuse")
+#         with gr.Column(scale=1):
+#             lngbtn = gr.Button("Synthesize", variant="primary")
+#             lngaudio = gr.Audio(interactive=False, label="Synthesized Audio")
+#             lngbtn.click(longsynthesize, inputs=[lnginp, lngvoice, lngsteps, lngpwd], outputs=[lngaudio], concurrency_limit=4)
+with gr.Blocks() as lj:
+    with gr.Row():
+        with gr.Column(scale=1):
+            ljinp = gr.Textbox(label="Text", info="What would you like StyleTTS 2 to read? It works better on full sentences.", interactive=True)
+            ljsteps = gr.Slider(minimum=3, maximum=20, value=3, step=1, label="Diffusion Steps", info="Theoretically, higher should be better quality but slower, but we cannot notice a difference. Try with lower steps first - it is faster", interactive=True)
+        with gr.Column(scale=1):
+            ljbtn = gr.Button("Synthesize", variant="primary")
+            ljaudio = gr.Audio(interactive=False, label="Synthesized Audio", waveform_options={'waveform_progress_color': '#3C82F6'})
+            ljbtn.click(ljsynthesize, inputs=[ljinp, ljsteps], outputs=[ljaudio], concurrency_limit=4)
+with gr.Blocks(title="StyleTTS 2", css="footer{display:none !important}", theme=theme) as demo:
+    gr.Markdown(INTROTXT)
+    gr.DuplicateButton("Duplicate Space")
+    # gr.TabbedInterface([vctk, clone, lj, longText], ['Multi-Voice', 'Voice Cloning', 'LJSpeech', 'Long Text [Beta]'])
+    gr.TabbedInterface([vctk, clone, lj], ['Multi-Voice', 'Voice Cloning', 'LJSpeech', 'Long Text [Beta]'])
+    gr.Markdown("""
+Demo by [mrfakename](https://twitter.com/realmrfakename). I am not affiliated with the StyleTTS 2 authors.
+
+Run this demo locally using Docker:
+
+```bash
+docker run -it -p 7860:7860 --platform=linux/amd64 --gpus all registry.hf.space/styletts2-styletts2:latest python app.py
+```
+""") # Please do not remove this line.
+if __name__ == "__main__":
+    # demo.queue(api_open=False, max_size=15).launch(show_api=False)
+    demo.queue(api_open=False, max_size=15).launch(show_api=False)
+

tts_harvard.py

@@ -1,48 +1,42 @@
-# Synthesize all Harvard Lists 77x lists of 10x sentences to single .wav ----- NEEDS TO BE RUN from https://github.com/audeering/shift/
+# Synthesize all Harvard Lists - 767 sentences as single .wav
 #
-# 1. using mimic3 style
-#     Folder: 'prompt_mimic3/'
-# 2. using mimic3 4x accelerated style
-#     Folder: 'prompt_mimic3speed/'
-# 3. using crema-d style
-#     Folder: 'prompt_human/'
+# 1.
 #
-# WAVS used from  tts_paper_plot.py
+#     './prompt_mimic3_english/'
+#
+# 2.
+#
+#     './prompt_mimic3_english_4x/'
+#
+# 3.
+#
+#     './prompt_human/'
+#
+# 4.
+#
+#     './prompt_mimic3_foreign/'
+#
+# 5.
+#   
+#     './prompt_mimic3_foreign_4x/
+#
+#
+#  ----> THE .wavs will be used for visualisation
 
 import soundfile
 import json
 import numpy as np
 import audb
 from pathlib import Path
-
+import os
 LABELS = ['arousal', 'dominance', 'valence']
 
 
-
-
-def load_speech(split=None):
+def load_human_speech(split=None):
     DB = [
         # [dataset, version, table, has_timdeltas_or_is_full_wavfile]
-           ['crema-d', '1.1.1', 'emotion.voice.test', False],
- #           ['emodb',  '1.2.0', 'emotion.categories.train.gold_standard', False],
-  #          ['entertain-playtestcloud', '1.1.0', 'emotion.categories.train.gold_standard', True],
-   #         ['erik', '2.2.0', 'emotion.categories.train.gold_standard', True],
-    #        ['meld', '1.3.1', 'emotion.categories.train.gold_standard', False],
-            # ['msppodcast', '5.0.0', 'emotion.categories.train.gold_standard', False],  # tandalone bucket because it has gt labels?
-     #       ['myai', '1.0.1', 'emotion.categories.train.gold_standard', False],
-      #      ['casia', None, 'emotion.categories.gold_standard', False],
-            # ['switchboard-1', None, 'sentiment', True],
-            # ['swiss-parliament', None, 'segments', True], 
-            # ['argentinian-parliament', None, 'segments', True],
-            # ['austrian-parliament', None, 'segments', True],
-            # #'german', --> bundestag
-            # ['brazilian-parliament', None, 'segments', True],
-            # ['mexican-parliament', None, 'segments', True],
-            # ['portuguese-parliament', None, 'segments', True],
-       #     ['spanish-parliament', None, 'segments', True],
-        #    ['chinese-vocal-emotions-liu-pell', None, 'emotion.categories.desired', False],
-            # peoples-speech slow
-         #   ['peoples-speech', None, 'train-initial', False]
+        #    ['crema-d', '1.1.1', 'emotion.voice.test', False],
+           ['emodb',  '1.2.0', 'emotion.categories.train.gold_standard', False],
     ]
 
     output_list = []
@@ -64,65 +58,38 @@ def load_speech(split=None):
             output_list += [f for f in a.index]  # use file (no timedeltas)
     return output_list
 
-
-
-
-
-# Generate 77 wavs
-
-
-
-
-with open('voices.json', 'r') as f:
-    df = json.load(f)['voices']
-voice_names = [v['voice'] for k,v in df.items()]
-synthetic_wav_paths = []
-synthetic_wav_paths_AFFECT = []
-for voice in voice_names:
     
-    synthetic_wav_paths.append(
-      'assets/wavs/style_vector/' + voice.replace('/', '_').replace('#', '_').replace(
-                    'cmu-arctic', 'cmu_arctic').replace('_low', '') + '.wav')
-    synthetic_wav_paths_AFFECT.append(
-        'assets/wavs/style_vector_v2/' + voice.replace('/', '_').replace('#', '_').replace(
-                    'cmu-arctic', 'cmu_arctic').replace('_low', '') + '.wav')
-
 
-print(len(synthetic_wav_paths))
-
-    
-natural_wav_paths = load_speech()
 
 
 # SYNTHESIZE mimic mimicx4 crema-d
 import msinference
-
+prompt_paths = {}
 
 with open('harvard.json', 'r') as f:
     harvard_individual_sentences = json.load(f)['sentences']
 
+for audio_prompt in [#'mimic3_english',
+                     #'mimic3_english_4x',
+                     'human',
+                     'mimic3_foreign',
+                     'mimic3_foreign_4x']:
+    
+    if audio_prompt == 'human':
+        prompt_paths = load_human_speech()  # better emodb ?
+    else:
+        prompt_dir = '/data/dkounadis/artificial-styletts2/' + audio_prompt + '/'
+        prompt_paths = [prompt_dir + f for f in os.listdir(prompt_dir)]
+    prompt_paths = prompt_paths[:10]
+    print(prompt_paths,'\n\n__________')
 
-
-
-
-for audio_prompt in ['mimic3', 'mimic3_speed', 'human']:
     total_audio = []
     ix = 0
-    for list_of_10 in harvard_individual_sentences:
+    for list_of_10 in harvard_individual_sentences[:1]:
         # long_sentence = ' '.join(list_of_10['sentences'])
         # harvard.append(long_sentence.replace('.', ' '))
         for text in list_of_10['sentences']:
-            if audio_prompt == 'mimic3':
-                style_vec = msinference.compute_style(
-                    synthetic_wav_paths[ix % 134])
-            elif audio_prompt == 'mimic3_speed':
-                style_vec = msinference.compute_style(
-                    synthetic_wav_paths_AFFECT[ix % 134])
-            elif audio_prompt == 'human':
-                style_vec = msinference.compute_style(
-                    natural_wav_paths[ix % len(natural_wav_paths)])
-            else:
-                print('unknonw list of style vecto')
+            style_vec = msinference.compute_style(prompt_paths[ix % len(prompt_paths)])
             print(ix, text)
             ix += 1
             x = msinference.inference(text,
@@ -133,10 +100,6 @@ for audio_prompt in ['mimic3', 'mimic3_speed', 'human']:
                                         embedding_scale=1)
             
             total_audio.append(x)
-            # concat before write
-        # -- for 10x sentenctes
-        print('_____________________')
-    # -- for 77x lists
-    total_audio = np.concatenate(total_audio)    
-    soundfile.write(f'{audio_prompt}_770.wav', total_audio, 24000)
-    print(f'{audio_prompt}_full_770.wav')
+    total_audio = np.concatenate(total_audio) # -- concat 77x lists
+    soundfile.write(f'{audio_prompt}_767_5.wav', total_audio, 24000)
+    print(f'{audio_prompt}_767_5.wav')

visualize_tts_plesantness.py

@@ -81,12 +81,12 @@ def _sigmoid(x):
 
 # for mimic3/mimic3speed/human - concat all 77 and run timeseries with 7s hop 3s
 for long_audio in [
-    # 'mimic3.wav',
-    #                 'mimic3_speedup.wav',
-                    'human_770.wav',  # 'mimic3_all_77.wav', #
-                    'mimic3_770.wav',
-                    'mimic3_speed_770.wav'
-                    ]:
+    'mimic3_english_767_5.wav',
+    'mimic3_english_4x_767_5.wav',
+    'human_767_5.wav',
+    'mimic3_foregin_767_5.wav',
+    'mimic3_foreign_4x_767_5.wav'
+    ]:
     file_interface = f'timeseries_{long_audio.replace("/", "")}.pkl'
     if not os.path.exists(file_interface):