visualize_tts_plesantness.py

# 1. engineer_style_foreign_style_vectors.py  # for speed=1 & speed=4
# 2. tts_harvard.py                           # (call inside SHIFT repo - needs StyleTTS msinference.py)
# 3. visualize_tts_pleasantness.py            # figures & audinterface


# Visualises timeseries 11 class for mimic3 human mimic3speed
#
#
# human_770.wav
# mimic3_770.wav
# mimic3_speedup_770.wav
FULL_WAV  = [
    'english_hfullh.wav',
    'english_4x_hfullh.wav',
    'human_hfullh.wav',
    'foreign_hfullh.wav',
    'foreign_4x_hfullh.wav',
                    ]
WIN = 40
HOP = 10
import pandas as pd
import os

import json
import numpy as np
import audonnx
import audb
from pathlib import Path
import transformers
import torch
import audmodel
import audinterface
import matplotlib.pyplot as plt
import audiofile

LABELS = ['arousal', 'dominance', 'valence',
        #    'speech_synthesizer', 'synthetic_singing',
           'Angry',
           'Sad',
           'Happy',
           'Surprise', 
            'Fear', 
            'Disgust', 
            'Contempt', 
            'Neutral'
            ]


config = transformers.Wav2Vec2Config() #finetuning_task='spef2feat_reg')
config.dev = torch.device('cuda:0')
config.dev2 = torch.device('cuda:0')

# def _softmax(x):
#     '''x : (batch, num_class)'''
#     x -= x.max(1, keepdims=True)  # if all -400 then sum(exp(x)) = 0
#     x = np.minimum(-100, x)
#     x = np.exp(x)
#     x /= x.sum(1, keepdims=True)
#     return x

def _softmax(x):
    '''x : (batch, num_class)'''
    x -= x.max(1, keepdims=True)  # if all -400 then sum(exp(x)) = 0
    x = np.maximum(-100, x)
    x = np.exp(x)
    x /= x.sum(1, keepdims=True)
    return x

def _sigmoid(x):
    '''x : (batch, num_class)'''
    return 1 / (1 + np.exp(-x))    


        # --
    # ALL = anger, contempt, disgust, fear, happiness, neutral, no_agreement, other, sadness, surprise
    # plot - unplesant emo 7x emo-categories [anger, contempt, disgust, fear, sadness] for artifical/sped-up/natural
    # plot - pleasant emo [neutral, happiness, surprise]
    # plot - Cubes Natural vs spedup   4x speed
    # plot - synthesizer class audioset


    # https://arxiv.org/pdf/2407.12229
    #  https://arxiv.org/pdf/2312.05187
    # https://arxiv.org/abs/2407.05407
    # https://arxiv.org/pdf/2408.06577
    # https://arxiv.org/pdf/2309.07405

    
# wavs are generated concat and plot time-series?

# for mimic3/mimic3speed/human - concat all 77 and run timeseries with 7s hop 3s
for long_audio in FULL_WAV:
    file_interface = f'timeseries_{long_audio.replace("/", "")}.pkl'
    if not os.path.exists(file_interface):


        print('_______________________________________\nProcessing\n', file_interface, '\n___________')



        # CAT MSP

        from transformers import AutoModelForAudioClassification
        import types
        def _infer(self, x):
            '''x: (batch, audio-samples-16KHz)'''
            x = (x + self.config.mean) / self.config.std  # plus
            x = self.ssl_model(x, attention_mask=None).last_hidden_state
            # pool
            h = self.pool_model.sap_linear(x).tanh()
            w = torch.matmul(h, self.pool_model.attention)
            w = w.softmax(1)
            mu = (x * w).sum(1)
            x = torch.cat(
                [
                    mu,
                    ((x * x * w).sum(1) - mu * mu).clamp(min=1e-7).sqrt()
                ], 1)
            return self.ser_model(x)

        teacher_cat = AutoModelForAudioClassification.from_pretrained(
            '3loi/SER-Odyssey-Baseline-WavLM-Categorical-Attributes',
            trust_remote_code=True  # fun definitions see 3loi/SER-.. repo
        ).to(config.dev2).eval()
        teacher_cat.forward = types.MethodType(_infer, teacher_cat)
        

        # ===================[:]===================== Dawn
        def _prenorm(x, attention_mask=None):
            '''mean/var'''
            if attention_mask is not None:
                N = attention_mask.sum(1, keepdim=True)  # here attn msk is unprocessed just the original input
                x -= x.sum(1, keepdim=True) / N
                var = (x * x).sum(1, keepdim=True) / N

            else:
                x -= x.mean(1, keepdim=True)  # mean is an onnx operator reducemean saves some ops compared to casting integer N to float and the div
                var = (x * x).mean(1, keepdim=True)
            return x / torch.sqrt(var + 1e-7)

        from torch import nn
        from transformers.models.wav2vec2.modeling_wav2vec2 import Wav2Vec2PreTrainedModel, Wav2Vec2Model
        class RegressionHead(nn.Module):
                r"""Classification head."""

                def __init__(self, config):

                    super().__init__()

                    self.dense = nn.Linear(config.hidden_size, config.hidden_size)
                    self.dropout = nn.Dropout(config.final_dropout)
                    self.out_proj = nn.Linear(config.hidden_size, config.num_labels)

                def forward(self, features, **kwargs):

                    x = features
                    x = self.dropout(x)
                    x = self.dense(x)
                    x = torch.tanh(x)
                    x = self.dropout(x)
                    x = self.out_proj(x)

                    return x


        class Dawn(Wav2Vec2PreTrainedModel):
            r"""Speech emotion classifier."""

            def __init__(self, config):

                super().__init__(config)

                self.config = config
                self.wav2vec2 = Wav2Vec2Model(config)
                self.classifier = RegressionHead(config)
                self.init_weights()

            def forward(
                    self,
                    input_values,
                    attention_mask=None,
            ):
                x = _prenorm(input_values, attention_mask=attention_mask)
                outputs = self.wav2vec2(x, attention_mask=attention_mask)
                hidden_states = outputs[0]
                hidden_states = torch.mean(hidden_states, dim=1)
                logits = self.classifier(hidden_states)
                return logits
                # return {'hidden_states': hidden_states,
                #         'logits': logits}
        dawn = Dawn.from_pretrained('audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim').to(config.dev).eval()
        # =======================================







        def process_function(x, sampling_rate, idx):
            '''run audioset ct, adv

                USE onnx teachers
                
                return [synth-speech, synth-singing, 7x, 3x adv] = 11
            '''
            
            # x = x[None , :]  ASaHSuFDCN
            #{0: 'Angry', 1: 'Sad', 2: 'Happy', 3: 'Surprise', 
            #4: 'Fear', 5: 'Disgust', 6: 'Contempt', 7: 'Neutral'}
            #tensor([[0.0015, 0.3651, 0.0593, 0.0315, 0.0600, 0.0125, 0.0319, 0.4382]])
            logits_cat = teacher_cat(torch.from_numpy(x).to(config.dev)).cpu().detach().numpy()
            # USE ALL CATEGORIES
            # --
            # logits_audioset = audioset_model(x, 16000)['logits_sounds']
            # logits_audioset = logits_audioset[:, [7, 35]]  # speech synthesizer synthetic singing
            # --
            logits_adv = dawn(torch.from_numpy(x).to(config.dev)).cpu().detach().numpy() #['logits']
            
            cat = np.concatenate([logits_adv,
                                #   _sigmoid(logits_audioset),
                                    _softmax(logits_cat)],
                                    1)
            print(cat)
            return cat #logits_adv #model(signal, sampling_rate)['logits']    


# ---------------------

        
        interface = audinterface.Feature(
            feature_names=LABELS,
            process_func=process_function,
            # process_func_args={'outputs': 'logits_scene'},
            process_func_applies_sliding_window=False,
            win_dur=WIN,
            hop_dur=HOP,
            sampling_rate=16000,
            resample=True,
            verbose=True,
        )
        df_pred = interface.process_file(long_audio)
        df_pred.to_pickle(file_interface)
    else:
        print(file_interface, 'FOUND')
        # df_pred = pd.read_pickle(file_interface)
        
        
        
# ===============================================================================
# V I S U A L S by loading all 3 pkl - mimic3 - speedup - human pd
#
# ===============================================================================


preds  = {}
SHORTEST_PD = 100000  # segments
for long_audio in FULL_WAV:
    file_interface = f'timeseries_{long_audio.replace("/", "")}.pkl'
    y = pd.read_pickle(file_interface)
    preds[long_audio] = y
    SHORTEST_PD = min(SHORTEST_PD, len(y))

# clean indexes for plot

for k,v in preds.items():
    p = v[:SHORTEST_PD]  # TRuncate extra segments - human is slower than mimic3
    # p = pd.read_pickle(student_file)
    p.reset_index(inplace= True)
    p.drop(columns=['file','start'], inplace=True)
    p.set_index('end', inplace=True)
    # p = p.filter(scene_classes) #['transport', 'indoor', 'outdoor'])
    p.index = p.index.map(mapper = (lambda x: x.total_seconds()))
    preds[k] = p

    # print(p, '\n\n\n\n \n')
    
print(preds.keys(),'p')  




# 2 PLOTS

for lang in ['english', 
             'foreign']:
            

            fig, ax = plt.subplots(nrows=8, ncols=2, figsize=(24,20.7),
                                   gridspec_kw={'hspace': 0, 'wspace': .04})


            

            time_stamp = preds['human_hfullh.wav'].index.to_numpy()
            for j, dim in enumerate(['arousal', 
                                    'dominance', 
                                    'valence']):

                # MIMIC3                      

                ax[j, 0].plot(time_stamp, preds[f'{lang}_hfullh.wav'][dim], 
                            color=(0,104/255,139/255), 
                            label='mean_1', 
                            linewidth=2)
                ax[j, 0].fill_between(time_stamp,

                                0*preds[f'{lang}_hfullh.wav'][dim],
                                preds['human_hfullh.wav'][dim],

                                color=(.2,.2,.2), 
                                alpha=0.244)
                if j == 0:
                    if lang == 'english':
                        desc = 'English'
                    else:
                        desc = 'Non-English'
                    ax[j, 0].legend([f'StyleTTS2 using Mimic-3 {desc}',
                                     f'StyleTTS2 uising EmoDB'], 
                                    prop={'size': 14},
                                    )
                ax[j, 0].set_ylabel(dim.lower(), color=(.4, .4, .4), fontsize=17)
                
                # TICK
                ax[j, 0].set_ylim([1e-7, .9999])
                # ax[j, 0].set_yticks([.25, .5,.75])
                # ax[j, 0].set_yticklabels(['0.25', '.5', '0.75'])
                ax[j, 0].set_xticklabels(['' for _ in ax[j, 0].get_xticklabels()])
                ax[j, 0].set_xlim([time_stamp[0], time_stamp[-1]])


            # MIMIC3   4x speed


                ax[j, 1].plot(time_stamp, preds[f'{lang}_4x_hfullh.wav'][dim], 
                            color=(0,104/255,139/255), 
                            label='mean_1', 
                            linewidth=2)
                ax[j, 1].fill_between(time_stamp,

                                0 * preds[f'{lang}_4x_hfullh.wav'][dim],
                                preds['human_hfullh.wav'][dim],

                                color=(.2,.2,.2),
                                alpha=0.244)
                if j == 0:
                    if lang == 'english':
                        desc = 'English'
                    else:
                        desc = 'Non-English'                 
                    ax[j, 1].legend([f'StyleTTS2 using Mimic-3 {desc} 4x speed',
                                    f'StyleTTS2 using EmoDB'], 
                                    prop={'size': 14}, 
                                    #  loc='lower right'
                                    )


                ax[j, 1].set_xlabel('720 Harvard Sentences')



                # TICK
                ax[j, 1].set_ylim([1e-7, .9999])
                # ax[j, 1].set_yticklabels(['' for _ in ax[j, 1].get_yticklabels()])
                ax[j, 1].set_xticklabels(['' for _ in ax[j, 0].get_xticklabels()])
                ax[j, 1].set_xlim([time_stamp[0], time_stamp[-1]])




                ax[j, 0].grid()
                ax[j, 1].grid()
            # CATEGORIE





            time_stamp = preds['human_hfullh.wav'].index.to_numpy()
            for j, dim in enumerate(['Angry', 
                                    'Sad',
                                    'Happy',
                                    #  'Surprise', 
                                    'Fear',
                                    'Disgust', 
                                    #  'Contempt',
                                    #  'Neutral'
                                    ]):   # ASaHSuFDCN
                j = j + 3  # skip A/D/V suplt                         

                # MIMIC3                      

                ax[j, 0].plot(time_stamp, preds[f'{lang}_hfullh.wav'][dim], 
                            color=(0,104/255,139/255), 
                            label='mean_1', 
                            linewidth=2)
                ax[j, 0].fill_between(time_stamp,

                                0*preds[f'{lang}_hfullh.wav'][dim],
                                preds['human_hfullh.wav'][dim],

                                color=(.2,.2,.2), 
                                alpha=0.244)
                # ax[j, 0].legend(['StyleTTS2 style mimic3',
                #                  'StyleTTS2 style crema-d'], 
                #                  prop={'size': 10}, 
                #                 #  loc='upper left'
                # )


                ax[j, 0].set_ylabel(dim.lower(), color=(.4, .4, .4), fontsize=17)

                # TICKS
                ax[j, 0].set_ylim([1e-7, .9999])
                ax[j, 0].set_xlim([time_stamp[0], time_stamp[-1]])
                ax[j, 0].set_xticklabels(['' for _ in ax[j, 0].get_xticklabels()])
                ax[j, 0].set_xlabel('720 Harvard Sentences', fontsize=17, color=(.2,.2,.2))


            # MIMIC3   4x speed


                ax[j, 1].plot(time_stamp, preds[f'{lang}_4x_hfullh.wav'][dim],
                            color=(0,104/255,139/255), 
                            label='mean_1', 
                            linewidth=2)
                ax[j, 1].fill_between(time_stamp,

                                0*preds[f'{lang}_4x_hfullh.wav'][dim],
                                preds['human_hfullh.wav'][dim],

                                color=(.2,.2,.2), 
                                alpha=0.244)
                # ax[j, 1].legend(['StyleTTS2 style mimic3   4x speed',
                #                  'StyleTTS2 style crema-d'], 
                #                  prop={'size': 10},
                #                 #  loc='upper left'
                # )
                ax[j, 1].set_xlabel('720 Harvard Sentences', fontsize=17, color=(.2,.2,.2))
                ax[j, 1].set_ylim([1e-7, .9999])
                # ax[j, 1].set_yticklabels(['' for _ in ax[j, 1].get_yticklabels()])
                ax[j, 1].set_xticklabels(['' for _ in ax[j, 1].get_xticklabels()])
                ax[j, 1].set_xlim([time_stamp[0], time_stamp[-1]])
                





                ax[j, 0].grid()
                ax[j, 1].grid()



            plt.savefig(f'fig_{lang}_{WIN=}_{HOP=}_fin0.pdf', bbox_inches='tight')
            plt.close()