# Synthesize all Harvard Lists 77x lists of 10x sentences to single .wav # 1. using mimic3 english 1x/4x non-english 1x/4x # Call visualize_tts_plesantness.py for 4figs [eng 1x/4x vs human, non-eng 1x/4x vs human-libri] import soundfile import json import numpy as np import audb from pathlib import Path LABELS = ['arousal', 'dominance', 'valence'] def load_speech(split=None): DB = [ # [dataset, version, table, has_timdeltas_or_is_full_wavfile] # ['crema-d', '1.1.1', 'emotion.voice.test', False], #['librispeech', '3.1.0', 'test-clean', False], ['emodb', '1.2.0', 'emotion.categories.train.gold_standard', False], # ['entertain-playtestcloud', '1.1.0', 'emotion.categories.train.gold_standard', True], # ['erik', '2.2.0', 'emotion.categories.train.gold_standard', True], # ['meld', '1.3.1', 'emotion.categories.train.gold_standard', False], # ['msppodcast', '5.0.0', 'emotion.categories.train.gold_standard', False], # tandalone bucket because it has gt labels? # ['myai', '1.0.1', 'emotion.categories.train.gold_standard', False], # ['casia', None, 'emotion.categories.gold_standard', False], # ['switchboard-1', None, 'sentiment', True], # ['swiss-parliament', None, 'segments', True], # ['argentinian-parliament', None, 'segments', True], # ['austrian-parliament', None, 'segments', True], # #'german', --> bundestag # ['brazilian-parliament', None, 'segments', True], # ['mexican-parliament', None, 'segments', True], # ['portuguese-parliament', None, 'segments', True], # ['spanish-parliament', None, 'segments', True], # ['chinese-vocal-emotions-liu-pell', None, 'emotion.categories.desired', False], # peoples-speech slow # ['peoples-speech', None, 'train-initial', False] ] output_list = [] for database_name, ver, table, has_timedeltas in DB: a = audb.load(database_name, sampling_rate=16000, format='wav', mixdown=True, version=ver, cache_root='/cache/audb/') a = a[table].get() if has_timedeltas: print(f'{has_timedeltas=}') # a = a.reset_index()[['file', 'start', 'end']] # output_list += [[*t] for t # in zip(a.file.values, a.start.dt.total_seconds().values, a.end.dt.total_seconds().values)] else: output_list += [f for f in a.index] # use file (no timedeltas) return output_list natural_wav_paths = load_speech() # SYNTHESIZE mimic mimicx4 crema-d import msinference import os from random import shuffle import audiofile with open('harvard.json', 'r') as f: harvard_individual_sentences = json.load(f)['sentences'] synthetic_wav_paths = ['./enslow/' + i for i in os.listdir('./enslow/')] synthetic_wav_paths_4x = ['./style_vector_v2/' + i for i in os.listdir('./style_vector_v2/')] synthetic_wav_paths_foreign = ['./mimic3_foreign/' + i for i in os.listdir('./mimic3_foreign/') if 'en_U' not in i] synthetic_wav_paths_foreign_4x = ['./mimic3_foreign_4x/' + i for i in os.listdir('./mimic3_foreign_4x/') if 'en_U' not in i] # very short segments # filter very short styles synthetic_wav_paths_foreign = [i for i in synthetic_wav_paths_foreign if audiofile.duration(i) > 2] synthetic_wav_paths_foreign_4x = [i for i in synthetic_wav_paths_foreign_4x if audiofile.duration(i) > 2] synthetic_wav_paths = [i for i in synthetic_wav_paths if audiofile.duration(i) > 2] synthetic_wav_pathsn_4x = [i for i in synthetic_wav_paths_4x if audiofile.duration(i) > 2] shuffle(synthetic_wav_paths_foreign_4x) shuffle(synthetic_wav_paths_foreign) shuffle(synthetic_wav_paths) shuffle(synthetic_wav_paths_4x) print(len(synthetic_wav_paths_foreign_4x), len(synthetic_wav_paths_foreign), len(synthetic_wav_paths), len(synthetic_wav_paths_4x)) # 134 204 134 204 for audio_prompt in ['english', 'english_4x', 'human', 'foreign', 'foreign_4x']: OUT_FILE = f'{audio_prompt}_hfullh.wav' if not os.path.isfile(OUT_FILE): total_audio = [] total_style = [] ix = 0 for list_of_10 in harvard_individual_sentences[:1000]: # long_sentence = ' '.join(list_of_10['sentences']) # harvard.append(long_sentence.replace('.', ' ')) for text in list_of_10['sentences']: if audio_prompt == 'english': _p = synthetic_wav_paths[ix % len(synthetic_wav_paths)] #134] style_vec = msinference.compute_style(_p) elif audio_prompt == 'english_4x': _p = synthetic_wav_paths_4x[ix % len(synthetic_wav_paths_4x)] # 134] style_vec = msinference.compute_style(_p) elif audio_prompt == 'human': _p = natural_wav_paths[ix % len(natural_wav_paths)] style_vec = msinference.compute_style(_p) elif audio_prompt == 'foreign': _p = synthetic_wav_paths_foreign[ix % len(synthetic_wav_paths_foreign)] #179] # 204 some short styles are discarded style_vec = msinference.compute_style(_p) elif audio_prompt == 'foreign_4x': _p = synthetic_wav_paths_foreign_4x[ix % len(synthetic_wav_paths_foreign_4x)] #179] # 204 style_vec = msinference.compute_style(_p) else: print('unknonw list of style vector') print(ix, text) ix += 1 x = msinference.inference(text, style_vec, alpha=0.3, beta=0.7, diffusion_steps=7, embedding_scale=1) total_audio.append(x) _st, fsr = audiofile.read(_p) total_style.append(_st[:len(x)]) # concat before write # -- for 10x sentenctes print('_____________________') # -- for 77x lists total_audio = np.concatenate(total_audio) soundfile.write(OUT_FILE, total_audio, 24000) total_style = np.concatenate(total_style) soundfile.write('_st_' + OUT_FILE, total_style, fsr) # take this fs from the loading else: print('\nALREADY EXISTS\n')