# 1. engineer_style_foreign_style_vectors.py # for speed=1 & speed=4 # 2. tts_harvard.py # (call inside SHIFT repo - needs StyleTTS msinference.py) # 3. visualize_tts_pleasantness.py # figures & audinterface # Visualises timeseries 11 class for mimic3 human mimic3speed # # # human_770.wav # mimic3_770.wav # mimic3_speedup_770.wav FULL_WAV = [ 'english_hfullh.wav', 'english_4x_hfullh.wav', 'human_hfullh.wav', 'foreign_hfullh.wav', 'foreign_4x_hfullh.wav', ] WIN = 40 HOP = 10 import pandas as pd import os import json import numpy as np import audonnx import audb from pathlib import Path import transformers import torch import audmodel import audinterface import matplotlib.pyplot as plt import audiofile LABELS = ['arousal', 'dominance', 'valence', # 'speech_synthesizer', 'synthetic_singing', 'Angry', 'Sad', 'Happy', 'Surprise', 'Fear', 'Disgust', 'Contempt', 'Neutral' ] config = transformers.Wav2Vec2Config() #finetuning_task='spef2feat_reg') config.dev = torch.device('cuda:0') config.dev2 = torch.device('cuda:0') # def _softmax(x): # '''x : (batch, num_class)''' # x -= x.max(1, keepdims=True) # if all -400 then sum(exp(x)) = 0 # x = np.minimum(-100, x) # x = np.exp(x) # x /= x.sum(1, keepdims=True) # return x def _softmax(x): '''x : (batch, num_class)''' x -= x.max(1, keepdims=True) # if all -400 then sum(exp(x)) = 0 x = np.maximum(-100, x) x = np.exp(x) x /= x.sum(1, keepdims=True) return x def _sigmoid(x): '''x : (batch, num_class)''' return 1 / (1 + np.exp(-x)) # -- # ALL = anger, contempt, disgust, fear, happiness, neutral, no_agreement, other, sadness, surprise # plot - unplesant emo 7x emo-categories [anger, contempt, disgust, fear, sadness] for artifical/sped-up/natural # plot - pleasant emo [neutral, happiness, surprise] # plot - Cubes Natural vs spedup 4x speed # plot - synthesizer class audioset # https://arxiv.org/pdf/2407.12229 # https://arxiv.org/pdf/2312.05187 # https://arxiv.org/abs/2407.05407 # https://arxiv.org/pdf/2408.06577 # https://arxiv.org/pdf/2309.07405 # wavs are generated concat and plot time-series? # for mimic3/mimic3speed/human - concat all 77 and run timeseries with 7s hop 3s for long_audio in FULL_WAV: file_interface = f'timeseries_{long_audio.replace("/", "")}.pkl' if not os.path.exists(file_interface): print('_______________________________________\nProcessing\n', file_interface, '\n___________') # CAT MSP from transformers import AutoModelForAudioClassification import types def _infer(self, x): '''x: (batch, audio-samples-16KHz)''' x = (x + self.config.mean) / self.config.std # plus x = self.ssl_model(x, attention_mask=None).last_hidden_state # pool h = self.pool_model.sap_linear(x).tanh() w = torch.matmul(h, self.pool_model.attention) w = w.softmax(1) mu = (x * w).sum(1) x = torch.cat( [ mu, ((x * x * w).sum(1) - mu * mu).clamp(min=1e-7).sqrt() ], 1) return self.ser_model(x) teacher_cat = AutoModelForAudioClassification.from_pretrained( '3loi/SER-Odyssey-Baseline-WavLM-Categorical-Attributes', trust_remote_code=True # fun definitions see 3loi/SER-.. repo ).to(config.dev2).eval() teacher_cat.forward = types.MethodType(_infer, teacher_cat) # ===================[:]===================== Dawn def _prenorm(x, attention_mask=None): '''mean/var''' if attention_mask is not None: N = attention_mask.sum(1, keepdim=True) # here attn msk is unprocessed just the original input x -= x.sum(1, keepdim=True) / N var = (x * x).sum(1, keepdim=True) / N else: x -= x.mean(1, keepdim=True) # mean is an onnx operator reducemean saves some ops compared to casting integer N to float and the div var = (x * x).mean(1, keepdim=True) return x / torch.sqrt(var + 1e-7) from torch import nn from transformers.models.wav2vec2.modeling_wav2vec2 import Wav2Vec2PreTrainedModel, Wav2Vec2Model class RegressionHead(nn.Module): r"""Classification head.""" def __init__(self, config): super().__init__() self.dense = nn.Linear(config.hidden_size, config.hidden_size) self.dropout = nn.Dropout(config.final_dropout) self.out_proj = nn.Linear(config.hidden_size, config.num_labels) def forward(self, features, **kwargs): x = features x = self.dropout(x) x = self.dense(x) x = torch.tanh(x) x = self.dropout(x) x = self.out_proj(x) return x class Dawn(Wav2Vec2PreTrainedModel): r"""Speech emotion classifier.""" def __init__(self, config): super().__init__(config) self.config = config self.wav2vec2 = Wav2Vec2Model(config) self.classifier = RegressionHead(config) self.init_weights() def forward( self, input_values, attention_mask=None, ): x = _prenorm(input_values, attention_mask=attention_mask) outputs = self.wav2vec2(x, attention_mask=attention_mask) hidden_states = outputs[0] hidden_states = torch.mean(hidden_states, dim=1) logits = self.classifier(hidden_states) return logits # return {'hidden_states': hidden_states, # 'logits': logits} dawn = Dawn.from_pretrained('audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim').to(config.dev).eval() # ======================================= def process_function(x, sampling_rate, idx): '''run audioset ct, adv USE onnx teachers return [synth-speech, synth-singing, 7x, 3x adv] = 11 ''' # x = x[None , :] ASaHSuFDCN #{0: 'Angry', 1: 'Sad', 2: 'Happy', 3: 'Surprise', #4: 'Fear', 5: 'Disgust', 6: 'Contempt', 7: 'Neutral'} #tensor([[0.0015, 0.3651, 0.0593, 0.0315, 0.0600, 0.0125, 0.0319, 0.4382]]) logits_cat = teacher_cat(torch.from_numpy(x).to(config.dev)).cpu().detach().numpy() # USE ALL CATEGORIES # -- # logits_audioset = audioset_model(x, 16000)['logits_sounds'] # logits_audioset = logits_audioset[:, [7, 35]] # speech synthesizer synthetic singing # -- logits_adv = dawn(torch.from_numpy(x).to(config.dev)).cpu().detach().numpy() #['logits'] cat = np.concatenate([logits_adv, # _sigmoid(logits_audioset), _softmax(logits_cat)], 1) print(cat) return cat #logits_adv #model(signal, sampling_rate)['logits'] # --------------------- interface = audinterface.Feature( feature_names=LABELS, process_func=process_function, # process_func_args={'outputs': 'logits_scene'}, process_func_applies_sliding_window=False, win_dur=WIN, hop_dur=HOP, sampling_rate=16000, resample=True, verbose=True, ) df_pred = interface.process_file(long_audio) df_pred.to_pickle(file_interface) else: print(file_interface, 'FOUND') # df_pred = pd.read_pickle(file_interface) # =============================================================================== # V I S U A L S by loading all 3 pkl - mimic3 - speedup - human pd # # =============================================================================== preds = {} SHORTEST_PD = 100000 # segments for long_audio in FULL_WAV: file_interface = f'timeseries_{long_audio.replace("/", "")}.pkl' y = pd.read_pickle(file_interface) preds[long_audio] = y SHORTEST_PD = min(SHORTEST_PD, len(y)) # clean indexes for plot for k,v in preds.items(): p = v[:SHORTEST_PD] # TRuncate extra segments - human is slower than mimic3 # p = pd.read_pickle(student_file) p.reset_index(inplace= True) p.drop(columns=['file','start'], inplace=True) p.set_index('end', inplace=True) # p = p.filter(scene_classes) #['transport', 'indoor', 'outdoor']) p.index = p.index.map(mapper = (lambda x: x.total_seconds())) preds[k] = p # print(p, '\n\n\n\n \n') print(preds.keys(),'p') # 2 PLOTS for lang in ['english', 'foreign']: fig, ax = plt.subplots(nrows=8, ncols=2, figsize=(24,20.7), gridspec_kw={'hspace': 0, 'wspace': .04}) time_stamp = preds['human_hfullh.wav'].index.to_numpy() for j, dim in enumerate(['arousal', 'dominance', 'valence']): # MIMIC3 ax[j, 0].plot(time_stamp, preds[f'{lang}_hfullh.wav'][dim], color=(0,104/255,139/255), label='mean_1', linewidth=2) ax[j, 0].fill_between(time_stamp, 0*preds[f'{lang}_hfullh.wav'][dim], preds['human_hfullh.wav'][dim], color=(.2,.2,.2), alpha=0.244) if j == 0: if lang == 'english': desc = 'English' else: desc = 'Non-English' ax[j, 0].legend([f'StyleTTS2 using Mimic-3 {desc}', f'StyleTTS2 uising EmoDB'], prop={'size': 14}, ) ax[j, 0].set_ylabel(dim.lower(), color=(.4, .4, .4), fontsize=17) # TICK ax[j, 0].set_ylim([1e-7, .9999]) # ax[j, 0].set_yticks([.25, .5,.75]) # ax[j, 0].set_yticklabels(['0.25', '.5', '0.75']) ax[j, 0].set_xticklabels(['' for _ in ax[j, 0].get_xticklabels()]) ax[j, 0].set_xlim([time_stamp[0], time_stamp[-1]]) # MIMIC3 4x speed ax[j, 1].plot(time_stamp, preds[f'{lang}_4x_hfullh.wav'][dim], color=(0,104/255,139/255), label='mean_1', linewidth=2) ax[j, 1].fill_between(time_stamp, 0 * preds[f'{lang}_4x_hfullh.wav'][dim], preds['human_hfullh.wav'][dim], color=(.2,.2,.2), alpha=0.244) if j == 0: if lang == 'english': desc = 'English' else: desc = 'Non-English' ax[j, 1].legend([f'StyleTTS2 using Mimic-3 {desc} 4x speed', f'StyleTTS2 using EmoDB'], prop={'size': 14}, # loc='lower right' ) ax[j, 1].set_xlabel('720 Harvard Sentences') # TICK ax[j, 1].set_ylim([1e-7, .9999]) # ax[j, 1].set_yticklabels(['' for _ in ax[j, 1].get_yticklabels()]) ax[j, 1].set_xticklabels(['' for _ in ax[j, 0].get_xticklabels()]) ax[j, 1].set_xlim([time_stamp[0], time_stamp[-1]]) ax[j, 0].grid() ax[j, 1].grid() # CATEGORIE time_stamp = preds['human_hfullh.wav'].index.to_numpy() for j, dim in enumerate(['Angry', 'Sad', 'Happy', # 'Surprise', 'Fear', 'Disgust', # 'Contempt', # 'Neutral' ]): # ASaHSuFDCN j = j + 3 # skip A/D/V suplt # MIMIC3 ax[j, 0].plot(time_stamp, preds[f'{lang}_hfullh.wav'][dim], color=(0,104/255,139/255), label='mean_1', linewidth=2) ax[j, 0].fill_between(time_stamp, 0*preds[f'{lang}_hfullh.wav'][dim], preds['human_hfullh.wav'][dim], color=(.2,.2,.2), alpha=0.244) # ax[j, 0].legend(['StyleTTS2 style mimic3', # 'StyleTTS2 style crema-d'], # prop={'size': 10}, # # loc='upper left' # ) ax[j, 0].set_ylabel(dim.lower(), color=(.4, .4, .4), fontsize=17) # TICKS ax[j, 0].set_ylim([1e-7, .9999]) ax[j, 0].set_xlim([time_stamp[0], time_stamp[-1]]) ax[j, 0].set_xticklabels(['' for _ in ax[j, 0].get_xticklabels()]) ax[j, 0].set_xlabel('720 Harvard Sentences', fontsize=17, color=(.2,.2,.2)) # MIMIC3 4x speed ax[j, 1].plot(time_stamp, preds[f'{lang}_4x_hfullh.wav'][dim], color=(0,104/255,139/255), label='mean_1', linewidth=2) ax[j, 1].fill_between(time_stamp, 0*preds[f'{lang}_4x_hfullh.wav'][dim], preds['human_hfullh.wav'][dim], color=(.2,.2,.2), alpha=0.244) # ax[j, 1].legend(['StyleTTS2 style mimic3 4x speed', # 'StyleTTS2 style crema-d'], # prop={'size': 10}, # # loc='upper left' # ) ax[j, 1].set_xlabel('720 Harvard Sentences', fontsize=17, color=(.2,.2,.2)) ax[j, 1].set_ylim([1e-7, .9999]) # ax[j, 1].set_yticklabels(['' for _ in ax[j, 1].get_yticklabels()]) ax[j, 1].set_xticklabels(['' for _ in ax[j, 1].get_xticklabels()]) ax[j, 1].set_xlim([time_stamp[0], time_stamp[-1]]) ax[j, 0].grid() ax[j, 1].grid() plt.savefig(f'fig_{lang}_{WIN=}_{HOP=}_fin0.pdf', bbox_inches='tight') plt.close()