<jupyter_start><jupyter_text>(Start) Testing Area<jupyter_code>import nltk
nltk.download('punkt')
from nltk import sent_tokenize
text = """
What the fuck did you just fucking say about me, you little pony?
I’ll have you know I graduated top of my class in magic kindergarten, and I’ve been involved in numerous secret raids on Nightmare Moon, and I have over 300 confirmed friendships.
I am trained in magic warfare and I’m the top pony in the entire Equestrian armed forces.
You are nothing to me but just another friend.
I will wipe you the fuck out with friendship the likes of which has never been seen before on Equestria, mark my fucking words.
You think you can get away with saying that shit to me over the Ponynet?
Think again, fucker.
As we speak I am contacting my secret network of pegasi across Equestria and your hoofprints are being traced right now so you better prepare for the storm, maggot.
The storm that wipes out the pathetic little thing you call your life. You’re fucking dead, pony. I can be anywhere, anytime, and I can hug you in over seven hundred ways, and that’s just with my bare hooves. Not only am I extensively trained in unarmed friendship, but I have access to the entire arsenal of ponies and I will use it to its full extent to wipe your miserable flank off the face of the continent, you little pony. If only you could have known what magical friendship your little “clever” comment was about to bring down upon you, maybe you would have held your fucking tongue. But you couldn’t, you didn’t, and now you’re paying the price, you goddamn pony. I will shit friendship all over you and you will drown in it. You’re fucking dead, pony.
"""
sentences = sent_tokenize(text)
print(sentences)
"adsoj   ".strip()
import random
x = [random.random() < 1.0 for i in range(100000)]
x = [y for y in x if y]
print(len(x))
from utils import get_mask_from_lengths
import torch
x = torch.tensor([1,5,6,2]).cuda().float()
print(x)
mask = get_mask_from_lengths(x, max_len=int(x.max().item()))
spec = torch.rand(4,160,)
print(   "s\r\nt".replace("\r\n","\n")   )
print( list(range(10)), "-"*30, sep="\n" )
#-----------------------------------------
print( list(range(10))[0::2] )
print( list(range(10))[1::2] )
2**7*8*2
2**7*8*2*(48000/22050)
a = ["a","b"]
b = [0,1]
c = dict(zip(a,b))
print(**c)
>>> m = weight_norm(nn.Linear(20, 40), name='weight')
>>> m
Linear(in_features=20, out_features=40, bias=True)
>>> m.weight_g.size()
torch.Size([40, 1])
>>> m.weight_v.size()
torch.Size([40, 20])<jupyter_output><empty_output><jupyter_text>Testing Performance impact of Weight norm<jupyter_code>import torch
import torch.nn as nn
import torch.nn.functional as F
x = torch.rand(8192, 8, 512, device="cuda")
linear_layer_wn = nn.utils.weight_norm(nn.Linear(512, 512), name='weight').cuda()
%%timeit
with torch.no_grad():
    for k in range(1):
        y = linear_layer(x)
linear_layer = nn.Linear(512, 512).cuda()
%%timeit
with torch.no_grad():
    for k in range(1):
        y = linear_layer(x)
print(
    hasattr(linear_layer, "weight"),
    hasattr(linear_layer, "weight_g"),
    hasattr(linear_layer_wn, "weight"),
    hasattr(linear_layer_wn, "weight_g"),
    linear_layer_wn.weight.shape,
    linear_layer_wn.weight_g.shape,
    linear_layer_wn.weight_v.shape,
    sep="\n")
list(linear_layer_wn.named_modules())[0][1].weight
list(range(5))[:-1]
import torch
audio = (torch.rand(10)-0.5)*2
audio[audio>0] = 2**(audio[audio>0].log10())
audio[audio<0] = -(2**((-audio[audio<0]).log10()))
audio
1/(8.26*10**-1)
import os
import numpy as np
import random
import sys
import time
import argparse
import torch
import matplotlib.pyplot as plt
from scipy.io.wavfile import write
from model import Tacotron2
from train import load_model
from text import text_to_sequence
from denoiser import Denoiser
from utils import load_filepaths_and_text
import json
import re
import difflib
from glob import glob
from unidecode import unidecode
import nltk # sentence spliting
from nltk import sent_tokenize
waveglow_path = r"H:\TTCheckpoints\waveglow\outdir_EfficientSmallGlobalSpeakerEmbeddings\Testing2\best_val_weights"
config_fpath = r"H:\TTCheckpoints\waveglow\outdir_EfficientSmallGlobalSpeakerEmbeddings\Testing2\config.json"

def is_ax(config):
    """Quickly check if a model uses the Ax WaveGlow core by what's available in the config file."""
    return True if 'upsample_first' in config.keys() else False

# Load config file
with open(config_fpath) as f:
    data = f.read()
config = json.loads(data)
train_config = config["train_config"]
data_config = config["data_config"]
dist_config = config["dist_config"]
waveglow_config = {
    **config["waveglow_config"], 
    'win_length': data_config['win_length'],
    'hop_length': data_config['hop_length']
}
print(waveglow_config)
print(f"Config File from '{config_fpath}' successfully loaded.")

# import the correct model core
if is_ax(waveglow_config):
    from efficient_model_ax import WaveGlow
else:
    if waveglow_config["yoyo"]:
        from efficient_model import WaveGlow
    else:
        from glow import WaveGlow

# initialize model
print(f"intializing WaveGlow model... ", end="")
waveglow = WaveGlow(**waveglow_config).cuda()
print(f"Done!")

# load checkpoint from file
print(f"loading WaveGlow checkpoint... ", end="")
checkpoint = torch.load(waveglow_path)
waveglow.load_state_dict(checkpoint['model']) # and overwrite initialized weights with checkpointed weights
waveglow.cuda().eval().half() # move to GPU and convert to half precision
print(f"Done!")

print(f"initializing Denoiser... ", end="")
denoiser = Denoiser(waveglow)
print(f"Done!")
waveglow_iters = checkpoint['iteration']
print(f"WaveGlow trained for {waveglow_iters} iterations")
speaker_lookup = checkpoint['speaker_lookup'] # ids lookup
training_sigma = train_config['sigma']<jupyter_output>{'yoyo': True, 'yoyo_WN': False, 'n_mel_channels': 160, 'n_flows': 16, 'n_group': 120, 'n_early_every': 16, 'n_early_size': 2, 'memory_efficient': False, 'spect_scaling': False, 'upsample_mode': 'normal', 'WN_config': {'n_layers': 5, 'n_channels': 384, 'kernel_size': 3, 'speaker_embed_dim': 96, 'rezero': False}, 'win_length': 2400, 'hop_length': 600}
Config File from 'H:\TTCheckpoints\waveglow\outdir_EfficientSmallGlobalSpeakerEmbeddings\Testing2\config.json' successfully loaded.
intializing WaveGlow model... Done!
loading WaveGlow checkpoint... Done!
initializing Denoiser... Done!
WaveGlow trained for 282000 iterations<jupyter_text>---------------------------------------------------------<jupyter_code>sample_ratio = (3*38400) / (24*18000)
time_per_iter_ratio = (0.65) / (2.32)
source_lr = 4e-4
print( f"{source_lr*sample_ratio:.8f}" )
print( sample_ratio )
print( time_per_iter_ratio )
import torch
torch.rand(5).half().dtype
# TODO, invert this guy. Instead of splitting chunks of text that go over a limit. Split ALL the text at the start, and keep merging the text into chunks that are as large as possible!
def parse_text_into_quotes(texts):
    """Swap speaker at every quote mark. Also split longer phrases at periods.
    If text previously inside a quote is too long and split, then each split segment will have quotes around it (for information later on rather than accuracy to the original text)."""
    max_text_segment_length = 120
    quo ='"' # nested quotes in list comprehension are hard to work with
    texts = [f'"{text.replace(quo,"").strip()}"' if i%2 else text.replace(quo,"").strip() for i, text in enumerate(unidecode(texts).split('"'))]
    
    texts_segmented = []
    for text in texts:
        text = text.strip()
        if not len(text.replace('"','').strip()): continue
        text = text\
            .replace("\n"," ")\
            .replace("  "," ")\
            .replace("> --------------------------------------------------------------------------","")
        if len(text) > max_text_segment_length:
            for seg in [x.strip() for x in sent_tokenize(text) if len(x.strip()) and x is not '"']: 
                if '"' in text:
                    if seg[0] != '"': seg='"'+seg
                    if seg[-1] != '"': seg+='"'
                texts_segmented.append(seg)
        else:
            texts_segmented.append(text.strip())
    return texts_segmented
from unidecode import unidecode
import nltk # sentence spliting
from nltk import sent_tokenize
# generator for text splitting.
def parse_text_into_segments(texts, split_at_quotes=True, target_segment_length=200):
    """Swap speaker at every quote mark. Each split segment will have quotes around it (for information later on rather than accuracy to the original text)."""
    
    # split text by quotes
    quo ='"' # nested quotes in list comprehension are hard to work with
    texts = [f'"{text.replace(quo,"").strip()}"' if i%2 else text.replace(quo,"").strip() for i, text in enumerate(unidecode(texts).split('"'))]
    
    # clean up and remove empty texts
    def clean_text(text):
        text = text.strip()
        text = text.replace("\n"," ").replace("  "," ").replace("> --------------------------------------------------------------------------","")
        return text
    texts = [clean_text(text) for text in texts if len(text.strip().replace('"','').strip()) or len(clean_text(text))]
    assert len(texts)
    
    # split text by sentences and add commas in where needed.
    def quotify(seg, text):
        if '"' in text:
            if seg[0] != '"': seg='"'+seg
            if seg[-1] != '"': seg+='"'
        return seg
    texts_tmp = []
    texts = [texts_tmp.extend([quotify(x.strip(), text) for x in sent_tokenize(text) if len(x.strip()) and x is not '"']) for text in texts]
    texts = texts_tmp
    del texts_tmp
    assert len(texts)
    
    # merge neighbouring sentences
    quote_mode = False
    texts_output = []
    texts_segmented = ''
    texts_len = len(texts)
    for i, text in enumerate(texts):
        # split segment if quote swap
        if split_at_quotes and ('"' in text and quote_mode == False) or (not '"' in text and quote_mode == True):
            texts_segmented.replace('""','')
            texts_output.append(texts_segmented)
            texts_segmented=text
            quote_mode = not quote_mode
        
        # split segment if max length
        elif len(texts_segmented+text) > target_segment_length:
            texts_segmented.replace('""','')
            texts_output.append(texts_segmented)
            texts_segmented=text
        
        else: # continue adding to segment
            texts_segmented+= f' {text}'
    # add any remaining stuff.
    if len(texts_segmented):
        texts_output.append(texts_segmented)
    assert len(texts_output)
    
    return texts_output
texts = """
Helloo everypony!
"""
parse_text_into_segments(texts, split_at_quotes=True, target_segment_length=200)<jupyter_output><empty_output><jupyter_text>Testing some built in pytorch distributions<jupyter_code>import torch
concentration1 = 0.1
concentration0 = 10+0.9
dist = torch.distributions.beta.Beta(concentration1, concentration0)
value = torch.rand(10)
print(value)
dist.log_prob(value=value)
import torch
k = 0# index
for k in range(11):
    a = 0.1
    b = 0.9
    n = 10
    concentration1 = k+a
    concentration0 = n-k+b
    dist = torch.distributions.beta.Beta(concentration1, concentration0)
    print(  dist.sample((5,5)).mean()  )
dist.sample((5,5)).mean()
# pi = log(P * ai-1)

# on average,
# energy[k] = (a*n)/(a+b)
(a*n)/(a+b)
# perfect<jupyter_output><empty_output><jupyter_text>TF overview of the DC Attention<jupyter_code># previous_alignments: [batch, enc_T]

# static convolution
previous_alignments = tf.expand_dims(previous_alignments, axis=2) # [batch, enc_T] -> [batch, enc_T, 1]
static_f = static_convolution(previous_alignments) # [batch, enc_T, 1] -> [batch, enc_T, attn_filters]
static_f = static_fc(static_f)  # [batch, enc_T, attn_filters] -> [batch, enc_T, attn_dim]

# dynamic convolution
dynamic_filters = tf.layers.dense(tf.layers.dense(query, 128, activation=tf.tanh, use_bias=True, name="dynamic_fc1"),
                                  21 * 8, use_bias=False, name="dynamic_fc2")
dynamic_filters = tf.reshape(dynamic_filters, [-1, 21, 8])
stacked_alignments = stack_alignments(previous_alignments)
dynamic_f = tf.matmul(stacked_alignments, dynamic_filters)
dynamic_f = dynamic_fc(dynamic_f) # [batch, enc_T, attn_dim]

# score
energy = compute_score(static_f, dynamic_f) # [batch, enc_T]

# prior bias
prior_filters = tf.convert_to_tensor(
    [0.7400209, 0.07474979, 0.04157422, 0.02947039, 0.023170564, 0.019321883,
     0.016758798, 0.014978543, 0.013751862, 0.013028075, 0.013172861], dtype=tf.float32)
prior_filters = tf.reverse(prior_filters, axis=[0])
prior_filters = tf.reshape(prior_filters, [11, 1, 1])

bias = tf.nn.conv1d(tf.pad(previous_alignments, [[0, 0], [10, 0], [0, 0]]),
                    prior_filters, stride=1, padding='VALID')
bias = tf.maximum(tf.log(tf.squeeze(bias, axis=2)), -1.0e6)

energy += bias
alignments = _probability_fn(energy)  # softmax<jupyter_output><empty_output><jupyter_text>and converted to Pytorch<jupyter_code># previous_alignments: [batch, enc_T]

# static convolution
previous_alignments = previous_alignments[..., None]
static_f = self.location_layer(previous_alignments) # [B, 2, enc_T] -> [B, attention_n_filters, enc_T] -> [B, enc_T, attention_dim]

# dynamic convolution
dynamic_filters = torch.nn.Sequential(
    nn.Linear(query, 128 , bias=True), # attention dim
    nn.Tanh()
    nn.Linear(128  , 21*8, bias=False) # filter_num * filter_length
    )

dynamic_filters = dynamic_filters.reshape([-1, 21, 8]) # [batch, filter_num, filter_length]
stacked_alignments = previous_alignments.clone() #torch.stack(previous_alignments, dim=0) # which dim?
dynamic_f = stacked_alignments @ dynamic_filters # [batch, enc_T] @ [batch, filter_num, filter_length] ->  [batch, enc_T, attn_dim]
dynamic_f = self.vg(dynamic_f) # [batch, enc_T, attn_dim] -> [batch, enc_T, 1]

# score # WTF does this do?
energy = compute_score(static_f, dynamic_f) # [batch, enc_T]

# prior bias
prior_filters = torch.tensor(
    [0.7400209, 0.07474979, 0.04157422, 0.02947039, 0.023170564, 0.019321883,
     0.016758798, 0.014978543, 0.013751862, 0.013028075, 0.013172861])
prior_filters = prior_filters.flip(dims=(0,))
prior_filters = prior_filters[:, 1, 1]

previous_alignments = F.pad(previous_alignments, [[0, 0], [10, 0], [0, 0]])
bias = nn.conv1d(previous_alignments, prior_filters, stride=1, padding='VALID')
bias = torch.clamp(bias.squeeze(2).log(), min=-1.0e6)

energy += bias
alignments = _probability_fn(energy)  # softmax
{k:k for k in range(512)}<jupyter_output><empty_output><jupyter_text>example Prior Filter code in TF<jupyter_code>import tensorflow as tf
import sys
prior_filters = tf.convert_to_tensor(
    [0.7400209, 0.07474979, 0.04157422, 0.02947039, 0.023170564, 0.019321883,
     0.016758798, 0.014978543, 0.013751862, 0.013028075, 0.013172861], dtype=tf.float32)
# [11]

prior_filters = tf.reverse(prior_filters, axis=[0]) # [11]
prior_filters = tf.reshape(prior_filters, [11, 1, 1]) # [11, 1, 1]
print( prior_filters.eval(session=tf.compat.v1.Session()) )

prev_alignment = tf.one_hot([0], 60) # [1, 60] blank previous alignments

print( prev_alignment.eval(session=tf.compat.v1.Session()) )

for i in range(1):
    expanded_alignment = tf.expand_dims(prev_alignment, axis=2) # [1, 60, 1]
    expanded_alignment = tf.pad(expanded_alignment, [[0, 0], [10, 0], [0, 0]]) # [1, 70, 1]
    energy = tf.nn.conv1d(expanded_alignment, prior_filters, stride=1, padding='VALID') # [1, 70, 1] -> [1, 60, 1] prior energy
    energy = tf.squeeze(energy, axis=2) # [1, 60, 1] ->[1, 60]
    energy = tf.log(energy) # [1, 60] -> [1, 60]
    energy = tf.maximum(energy, -1.0e6)# [1, 60] -> [1, 60]
    alignment = tf.nn.softmax(energy, axis=-1) # [1, 60]
    prev_alignment = alignment # [1, 60]

alignment.eval(session=tf.compat.v1.Session())<jupyter_output>[[[0.01317286]]

 [[0.01302807]]

 [[0.01375186]]

 [[0.01497854]]

 [[0.0167588 ]]

 [[0.01932188]]

 [[0.02317056]]

 [[0.02947039]]

 [[0.04157422]]

 [[0.07474979]]

 [[0.7400209 ]]]
[[1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]]<jupyter_text>and converted to Pytorch<jupyter_code>import torch
import torch.nn.functional as F
prior_filters = torch.tensor(
    [0.7400209, 0.07474979, 0.04157422, 0.02947039, 0.023170564, 0.019321883,
     0.016758798, 0.014978543, 0.013751862, 0.013028075, 0.013172861])
# [11]

prior_filters = prior_filters.flip(dims=(0,)) # [11]
prior_filters = prior_filters[None, None, :] # [1, 1, 11]
print(prior_filters)

prev_alignment = torch.tensor([0.,]*60)[None, None, :] # [60] blank previous alignments
prev_alignment[:,:,0] = 1.0
print(prev_alignment)

for i in range(1):
    expanded_alignment = prev_alignment # [1, 60, 1]
    expanded_alignment = F.pad(expanded_alignment, (10, 0)) # [1, 70, 1]
    energy = F.conv1d(expanded_alignment, prior_filters) # [1, 70, 1] -> [1, 60, 1] prior energy
    energy = (energy).log()
    energy = torch.clamp(energy, min=-1.0e6)
    alignment = F.softmax(energy, dim=-1) # [1, 60]
    prev_alignment = alignment # [1, 60]

print(alignment)
len ( [0.7400209, 0.07474979, 0.04157422, 0.02947039, 0.023170564, 0.019321883, 0.016758798, 0.014978543, 0.013751862, 0.013028075, 0.013172861] )<jupyter_output><empty_output><jupyter_text>Some Comparisons between Depthwise Convs and Normal Convs<jupyter_code>segment_length = 24000

norm_n_flows = 12
norm_n_group = 20
norm_n_layers = 9
norm_n_channels = 256
norm_kernel_width = 3

sep_n_flows = 12
sep_n_group = 20
sep_n_layers = 9
sep_n_channels = 384
sep_kernel_width = 3
####################################################################
norm_mac = (norm_kernel_width*norm_n_channels**2) * (segment_length//norm_n_group)
norm_par = (norm_kernel_width*norm_n_channels**2)
sep_mac = ((sep_n_channels**2)+(sep_kernel_width*sep_n_channels)) * (segment_length//sep_n_group)
sep_par = ((sep_n_channels**2)+(sep_kernel_width*sep_n_channels))
###
norm_mac *= norm_n_layers*norm_n_flows
norm_par *= norm_n_layers*norm_n_flows
sep_mac *= sep_n_layers*sep_n_flows
sep_par *= sep_n_layers*sep_n_flows
####################################################################
print("Normal Conv MACs     = ", norm_mac/1e9, "B", sep="")
print("Seperable Conv MACs  = ", sep_mac/1e9, "B", sep="")
print("")
print("Normal Conv Params   = ", norm_par/1e6, "M", sep="")
print("SeperableConv Params = ", sep_par/1e6, "M", sep="")
print("")
print("Norn/Sep MACs =", round(norm_mac/sep_mac, 2))
print("Norn/Sep Params =", round(norm_mac/sep_mac, 2))
import torch
x = torch.rand(5,5)
y = torch.rand(5,1)
print(x.shape)
print(y.shape)
print(x)
print(y)
print(x*y)
print(x@y)
sum([x*2 for x in [1,1,2,1,1,2,1,1]])+1<jupyter_output><empty_output><jupyter_text>Get receptive field of WaveGlow/WaveFlow<jupyter_code>n_flows = 6
n_group = 50
kernel_size = 5
n_layers = 7


r_field = ( n_group*(2**(n_layers-1))*(kernel_size-1) ) + 1
print(f"Receptive Field per Flow = {r_field}")
r_field_flows = (r_field//2*n_flows) + 1
print(f"Total Receptive Field = {r_field_flows}")
print(f"Closest Segment Length = {round(r_field_flows/600)*600}")
print(f"Closest Double Segment Length = {round(r_field_flows/300)*600}")<jupyter_output>Receptive Field per Flow = 12801
Total Receptive Field = 38401
Closest Segment Length = 38400
Closest Double Segment Length = 76800<jupyter_text>Doing some investigative state dicting<jupyter_code>import torch
checkpoint_path = r"H:\TTCheckpoints\waveglow\outdir_EfficientSmallGlobalSpeakerEmbeddings\Testing5\best_val_model"
checkpoint = torch.load(checkpoint_path)
state_dict = checkpoint['model']
for name, weights in state_dict.items():
    print(name)
    print(weights.shape)
    print("")<jupyter_output><empty_output><jupyter_text>(End) Testing Area  ----------------------------------  Resave Checkpoint without optimizer<jupyter_code>import torch
from os.path import exists

checkpoint_paths = [
    r"G:\TwiBot\tacotron2-PPP-1.3.0\outdir_truncated1\checkpoint_184000",
    r"G:\TwiBot\tacotron2-PPP-1.3.0\outdir_truncated1\checkpoint_186000",
    r"G:\TwiBot\tacotron2-PPP-1.3.0\outdir_truncated1\checkpoint_188000",
    r"G:\TwiBot\tacotron2-PPP-1.3.0\outdir_truncated1\checkpoint_190000",
    r"G:\TwiBot\tacotron2-PPP-1.3.0\outdir_truncated1\checkpoint_138000",
    ]

for checkpoint_path in checkpoint_paths:
    if not exists(checkpoint_path):
        print(f"{checkpoint_path}\nDoesn't exist. Skipping!\n")
        continue
    
    fpath = f"{checkpoint_path}_weights"
    
    if exists(fpath):
        print(f"{checkpoint_path}\nAlready Exists! Skipping!\n")
        continue
    
    print("Loading... ", end="")
    checkpoint = torch.load(checkpoint_path)
    checkpoint['optimizer'] = None
    
    print("Writing to file... ", end="")
    torch.save(checkpoint, fpath)
    del checkpoint
    print("Done!\n")
import torch
checkpoint_path = r"G:\TwiBot\tacotron2-PPP-1.3.0\outdir_truncated1\checkpoint_190000"
checkpoint = torch.load(checkpoint_path)
id_lookup = checkpoint['speaker_id_lookup']
print(checkpoint['speaker_id_lookup'].keys())
print(sorted(checkpoint['speaker_id_lookup'].keys()))
text = """|(Audiobook) Blizzard2011_Nancy|0
|(Audiodrama) Fallout Equestria_Littlepip|1
|(Game) Sonic_Blaze|2
|Postal Dude|3
|(News Extracts) VCTK_P306|4
|(News Extracts) VCTK_P312|5
|(News Extracts) VCTK_P292|6
|(News Extracts) VCTK_P234|7
|(News Extracts) VCTK_P265|8
|(News Extracts) VCTK_P277|9
|(News Extracts) VCTK_P300|10
|(News Extracts) VCTK_P225|11
|(News Extracts) VCTK_P258|12
|(News Extracts) VCTK_P230|13
|(News Extracts) VCTK_P304|14
|(News Extracts) VCTK_P228|15
|(News Extracts) VCTK_P282|16
|(News Extracts) VCTK_P239|17
|(News Extracts) VCTK_P253|18
|(News Extracts) VCTK_P316|19
|(News Extracts) VCTK_P339|20
|(News Extracts) VCTK_P252|21
|(News Extracts) VCTK_P236|22
|(News Extracts) VCTK_P286|23
|(News Extracts) VCTK_P232|24
|(News Extracts) VCTK_P251|25
|(News Extracts) VCTK_P279|26
|(News Extracts) VCTK_P238|27
|(News Extracts) VCTK_P311|28
|(News Extracts) VCTK_P262|29
|(News Extracts) VCTK_P341|30
|(News Extracts) VCTK_P298|31
|(News Extracts) VCTK_P271|32
|(News Extracts) VCTK_P294|33
|(News Extracts) VCTK_P272|34
|(News Extracts) VCTK_P329|35
|(News Extracts) VCTK_P336|36
|(News Extracts) VCTK_P314|37
|(News Extracts) VCTK_P308|38
|(News Extracts) VCTK_P362|39
|(News Extracts) VCTK_P254|40
|(News Extracts) VCTK_P307|41
|(News Extracts) VCTK_P340|42
|(News Extracts) VCTK_P226|43
|(News Extracts) VCTK_P274|44
|(News Extracts) VCTK_P323|45
|(News Extracts) VCTK_P335|46
|(News Extracts) VCTK_P243|47
|(News Extracts) VCTK_P240|48
|(News Extracts) VCTK_P285|49
|(News Extracts) VCTK_P237|50
|(News Extracts) VCTK_P288|51
|(News Extracts) VCTK_P302|52
|(News Extracts) VCTK_P248|53
|(News Extracts) VCTK_P227|54
|(News Extracts) VCTK_P347|55
|(News Extracts) VCTK_P231|56
|(News Extracts) VCTK_P264|57
|(News Extracts) VCTK_P278|58
|(News Extracts) VCTK_P250|59
|(News Extracts) VCTK_P360|60
|(News Extracts) VCTK_P263|61
|(News Extracts) VCTK_P261|62
|(News Extracts) VCTK_P229|63
|(News Extracts) VCTK_P376|64
|(News Extracts) VCTK_P269|65
|(News Extracts) VCTK_P268|66
|(News Extracts) VCTK_P284|67
|(News Extracts) VCTK_P257|68
|(News Extracts) VCTK_P343|69
|(News Extracts) VCTK_P267|70
|(News Extracts) VCTK_P361|71
|(News Extracts) VCTK_P244|72
|(News Extracts) VCTK_P266|73
|(News Extracts) VCTK_P330|74
|(News Extracts) VCTK_P283|75
|(News Extracts) VCTK_P313|76
|(News Extracts) VCTK_P318|77
|(News Extracts) VCTK_P233|78
|(News Extracts) VCTK_P270|79
|(News Extracts) VCTK_P363|80
|(News Extracts) VCTK_P303|81
|(News Extracts) VCTK_P326|82
|(News Extracts) VCTK_P334|83
|(News Extracts) VCTK_P276|84
|(News Extracts) VCTK_P241|85
|(News Extracts) VCTK_P245|86
|(News Extracts) VCTK_P275|87
|(News Extracts) VCTK_P310|88
|(News Extracts) VCTK_P317|89
|(News Extracts) VCTK_P259|90
|(News Extracts) VCTK_P256|91
|(News Extracts) VCTK_P374|92
|(News Extracts) VCTK_P247|93
|(News Extracts) VCTK_P333|94
|(News Extracts) VCTK_P260|95
|(News Extracts) VCTK_P297|96
|(News Extracts) VCTK_P301|97
|(News Extracts) VCTK_P351|98
|(News Extracts) VCTK_P280|99
|(News Extracts) VCTK_P255|100
|(News Extracts) VCTK_P287|101
|(News Extracts) VCTK_P293|102
|(News Extracts) VCTK_P246|103
|(News Extracts) VCTK_P299|104
|(News Extracts) VCTK_P281|105
|(News Extracts) VCTK_P273|106
|(News Extracts) VCTK_P305|107
|(News Extracts) VCTK_P345|108
|(News Extracts) VCTK_P295|109
|(News Extracts) VCTK_P364|110
|(News Extracts) VCTK_P249|111
|(Music) My Little Pony_Rarity|112
|(Music) My Little Pony_Sunset Shimmer|113
|(Music) My Little Pony_Twilight|114
|(Music) My Little Pony_Mane 6|115
|(Music) My Little Pony_Luna|116
|(Music) My Little Pony_Autumn Blaze|117
|(Music) My Little Pony_Pinkie|118
|(Music) My Little Pony_Applejack|119
|(Music) My Little Pony_Scootaloo|120
|(Music) My Little Pony_Gloriosa Daisy|121
|(Music) My Little Pony_Cheese Sandwich|122
|(Music) My Little Pony_Spike|123
|(Music) My Little Pony_Trixie|124
|(Music) My Little Pony_Sludge|125
|(Music) My Little Pony_Apple Bloom|126
|(Music) My Little Pony_Adagio Dazzle|127
|(Music) My Little Pony_Flim|128
|(Music) My Little Pony_Flim Flam|129
|(Music) My Little Pony_Wallflower Blush|130
|(Music) My Little Pony_Rainbow|131
|(Music) My Little Pony_Rumble|132
|(Music) My Little Pony_Fluttershy|133
|(Music) My Little Pony_Diamond Tiara|134
|(Music) My Little Pony_Countess Coloratura|135
|(Music) My Little Pony_Feather Bangs|136
|(Music) My Little Pony_Cadance|137
|(Music) My Little Pony_Cmc|138
|(Music) My Little Pony_Starlight|139
|(Music) My Little Pony_Celestia|140
|(Music) My Little Pony_Cozy Glow|141
|(Music) My Little Pony_Tirek|142
|(Music) My Little Pony_Sweetie Belle|143
|(Music) My Little Pony_Chrysalis|144
|(Music) My Little Pony_Dazzlings|145
|(Music) My Little Pony_Pear Butter|146
|(Music) My Little Pony_Big Mac|147
|(Music) My Little Pony_Gabby|148
|(Audiobook) Sum - Tales From The Afterlives_Blunt|149
|(Game) Them's Fightin' Herds_Oleander|150
|(Game) Them's Fightin' Herds_Fred|151
|(Game) Them's Fightin' Herds_Pom|152
|(Game) Them's Fightin' Herds_Velvet|153
|(Game) Them's Fightin' Herds_Arizona|154
|(Game) Them's Fightin' Herds_Tianhuo|155
|(Game) Elite Dangerous_Eli|156
|(Audiobook) A Little Bit Wicked_Skystar|157
|(Audiobook) Dr. Who_Doctor|158
|(Show) My Little Pony_Applejack|159
|(Show) My Little Pony_Rainbow|160
|(Show) My Little Pony_Pinkie|161
|(Show) My Little Pony_Rarity|162
|(Show) My Little Pony_Spike|163
|(Show) My Little Pony_Fluttershy|164
|(Show) My Little Pony_Nightmare Moon|165
|(Show) Dan Vs_Dan|166
|(Show) My Little Pony_Twilight|167
|(Show) My Little Pony_Scootaloo|168
|(Show) My Little Pony_Big Mac|169
|(Show) My Little Pony_Sweetie Belle|170
|(Show) My Little Pony_Apple Bloom|171
|(Show) My Little Pony_Discord|172
|(Show) My Little Pony_Mrs. Cake|173
|(Show) My Little Pony_Sugar Belle|174
|(Show) My Little Pony_Granny Smith|175
|(Show) My Little Pony_Mayor Mare|176
|(Show) My Little Pony_Derpy|177
|(Show) My Little Pony_Cherry Jubilee|178
|(Show) My Little Pony_Sky Stinger|179
|(Show) My Little Pony_Vapor Trail|180
|(Show) My Little Pony_Spitfire|181
|(Show) My Little Pony_Stellar Flare|182
|(Show) My Little Pony_Sunburst|183
|(Show) My Little Pony_Starlight|184
|(Show) My Little Pony_Firelight|185
|(Show) My Little Pony_Jack Pot|186
|(Show) My Little Pony_Soarin|187
|(Show) My Little Pony_Goldie Delicious|188
|(Show) My Little Pony_Apple Rose|189
|(Show) My Little Pony_Auntie Applesauce|190
|(Show) My Little Pony_Celestia|191
|(Show) My Little Pony_On Stage|192
|(Show) My Little Pony_Sunset Shimmer|193
|(Show) My Little Pony_Vignette|194
|(Show) My Little Pony_Sky Beak|195
|(Show) My Little Pony_Terramar|196
|(Show) My Little Pony_Ocean Flow|197
|(Show) My Little Pony_Smolder|198
|(Show) My Little Pony_Yona|199
|(Show) My Little Pony_Gallus|200
|(Show) My Little Pony_Ocellus|201
|(Show) My Little Pony_Sandbar|202
|(Show) My Little Pony_Silverstream|203
|(Show) My Little Pony_Zephyr|204
|(Show) My Little Pony_Snips|205
|(Show) My Little Pony_Luna|206
|(Show) My Little Pony_Flam|207
|(Show) My Little Pony_Flim|208
|(Show) My Little Pony_Photo Finish|209
|(Show) My Little Pony_Cranky|210
|(Show) My Little Pony_Zecora|211
|(Show) My Little Pony_Luster Dawn|212
|(Show) My Little Pony_Flash Sentry|213
|(Show) My Little Pony_Trixie|214
|(Show) My Little Pony_Cadance|215
|(Show) My Little Pony_Chrysalis|216
|(Show) My Little Pony_Tirek|217
|(Show) My Little Pony_Cozy Glow|218
|(Show) My Little Pony_Thorax|219
|(Show) My Little Pony_Spur|220
|(Show) My Little Pony_Biscuit|221
|(Show) My Little Pony_Octavia|222
|(Show) My Little Pony_Burnt Oak|223
|(Show) My Little Pony_Bright Mac|224
|(Show) My Little Pony_Pear Butter|225
|(Show) My Little Pony_Rockhoof|226
|(Show) My Little Pony_Grogar|227
|(Show) My Little Pony_Star Swirl|228
|(Show) My Little Pony_Hoo'Far|229
|(Show) My Little Pony_Autumn Blaze|230
|(Show) My Little Pony_Trouble Shoes|231
|(Show) My Little Pony_Braeburn|232
|(Show) My Little Pony_Mean Applejack|233
|(Show) My Little Pony_Mean Twilight Sparkle|234
|(Show) My Little Pony_Photographer|235
|(Show) My Little Pony_Mean Rarity|236
|(Show) My Little Pony_Lyra|237
|(Show) My Little Pony_Sweetie Drops|238
|(Show) My Little Pony_Dr. Hooves|239
|(Show) My Little Pony_Matilda|240
|(Show) My Little Pony_Steve|241
|(Show) My Little Pony_Sludge|242
|(Show) My Little Pony_Maud|243
|(Show) My Little Pony_Neighsay|244
|(Show) My Little Pony_Ember|245
|(Show) My Little Pony_Mudbriar|246
|(Show) My Little Pony_Limestone|247
|(Show) My Little Pony_Diamond Tiara|248
|(Show) My Little Pony_Babs Seed|249
|(Show) My Little Pony_Silver Spoon|250
|(Show) My Little Pony_Shining Armor|251
|(Show) My Little Pony_Miss Harshwhinny|252
|(Show) My Little Pony_Saffron Masala|253
|(Show) My Little Pony_Coriander Cumin|254
|(Show) My Little Pony_Zesty Gourmand|255
|(Show) My Little Pony_Gladmane|256
|(Show) My Little Pony_Bulk Biceps|257
|(Show) My Little Pony_Party Favor|258
|(Show) My Little Pony_Double Diamond|259
|(Show) My Little Pony_Filthy Rich|260
|(Show) My Little Pony_Gabby|261
|(Show) My Little Pony_Quibble Pants|262
|(Show) My Little Pony_A. <NAME>|263
|(Show) My Little Pony_Dr. Caballeron|264
|(Show) My Little Pony_Cheerilee|265
|(Show) My Little Pony_Mrs. Shy|266
|(Show) My Little Pony_Ahuizotl|267
|(Show) My Little Pony_Sapphire Shores|268
|(Show) My Little Pony_Cheese Sandwich|269
|(Show) My Little Pony_Iron Will|270
|(Show) My Little Pony_Sassy Saddles|271
|(Show) My Little Pony_Caballeron|272
|(Show) My Little Pony_Rumble|273
|(Show) My Little Pony_Prince Rutherford|274
|(Show) My Little Pony_Gilda|275
|(Show) My Little Pony_Moon Dancer|276
|(Show) My Little Pony_Mane Allgood|277
|(Show) My Little Pony_Snap Shutter|278
|(Show) My Little Pony_Fancy Pants|279
|(Show) My Little Pony_Wind Sprint|280
|(Show) My Little Pony_Clear Sky|281
|(Show) My Little Pony_Ma Hooffield|282
|(Show) My Little Pony_Big Daddy Mccolt|283
|(Show) My Little Pony_Pipsqueak|284
|(Show) My Little Pony_Coco Pommel|285
|(Show) My Little Pony_Countess Coloratura|286
|(Show) My Little Pony_Svengallop|287
|(Show) My Little Pony_Juniper Montage|288
|(Show) My Little Pony_Adagio Dazzle|289
|(Show) My Little Pony_Aria Blaze|290
|(Show) My Little Pony_Wallflower|291
|(Show) My Little Pony_Sour Sweet|292
|(Show) My Little Pony_Canter Zoom|293
|(Show) My Little Pony_Gloriosa Daisy|294
|(Show) My Little Pony_Timber Spruce|295
|(Show) My Little Pony_Cinch|296"""
text = text.split("\n")
text = [x.split(r"|") for x in text]
print("\n".join(sorted([x[1] for x in text])))
#text = [f'{a}|{b}|{id_lookup[int(c)]}' for a,b,c in text]
#text = "\n".join(text)
#print(text)
"Testing".split("_")[-1]
import torch
x = torch.rand(8,8192).cuda().half()
#x *= ((torch.arange(8)[None, :].repeat(5,1)).float() / 4)

def get_first_over_thresh(x, threshold=0.5):
    x[:,x.size(1)-1] = threshold # set last to threshold just incase the output didn't finish generating.
    x[x>threshold] = threshold
    return x.size(1)-(x.flip(dims=(1,)).argmax(dim=1))

def get_first_over_thresh(x, threshold):
    """Takes [B, T] and outputs first T over threshold for each B (output.shape = [B])."""
    x = x.cpu().float()
    x[:,-1] = threshold # set last to threshold just incase the output didn't finish generating.
    x[x>threshold] = threshold
    return (x.size(1)-1)-(x.flip(dims=(1,)).argmax(dim=1))

threshold = 0.9999

print(x.shape)
print(x)
torch.stack((
    get_first_over_thresh(x, 0.001),
    get_first_over_thresh(x, 0.5),
    get_first_over_thresh(x, 0.9),
    get_first_over_thresh(x, 0.9999),
    get_first_over_thresh(x, 1.0000),
    get_first_over_thresh(x, 1.1)), dim=0)
import torch
x = torch.rand(256, 8000).cuda()
device = x.device
print(device)
x = x.cpu()
print(x.device)
x = x.to(device)
print(x.device)
x.int()
import numpy as np
x = np.load("gate_batch_outputs.npy")
torch.from_numpy(x).float()
f"{0.486421986412986412:0.4f}"
import torch
def permute_channels(x, reverse=False, bipart=False, shift=False, inverse_shift=False):
    x = [x[:,i,:] for i in range(x.shape[1])]
    if bipart and reverse:
        half = len(x)//2
        x = x[:half][::-1] + x[half:][::-1] # reverse H halfs [0,1,2,3,4,5,6,7] -> [3,2,1,0] + [7,6,5,4] -> [3,2,1,0,7,6,5,4]
    elif reverse:
        x = x[::-1] # reverse entire H [0,1,2,3,4,5,6,7] -> [7,6,5,4,3,2,1,0]
    if shift:
        x = [x[-1],] + x[:-1] # shift last H into first position [0,1,2,3,4,5,6,7] -> [7,0,1,2,3,4,5,6]
    elif inverse_shift:
        x = x[1:] + [x[0],]   # shift first H into last position [0,1,2,3,4,5,6,7] -> [1,2,3,4,5,6,7,0]
    return torch.stack(x, dim=1)
x = torch.arange(20)[None,:,None]
print(x.squeeze())
x = permute_channels(x, True, True)
print(x.squeeze())
x = permute_channels(x, True, True)
print(x.squeeze())
x = permute_channels(x, True, False)
print(x.squeeze())
x = permute_channels(x, True, False)
print(x.squeeze())
import torch
import torch.nn.functional as F
x = torch.rand(6)
print(x)
samp = x[:1]
samp = x[1:2]
print(x)
x = list(range(5))
print(x[:1])
print(x[1:])
x = [None,]*8
print(x)
x[0] = torch.rand(5)
print(x)
n_layers = 8
n_layers_dilations_w = None
for i in range(n_layers):
    dilation_w = 2 ** i if n_layers_dilations_w is None else n_layers_dilations_w[i]
    print(dilation_w)<jupyter_output>1
2
4
8
16
32
64
128<jupyter_text>----------------------------------  Tacotron 2 inference code Edit the variables **checkpoint_path** and **text** to match yours and run the entire code to generate plots of mel outputs, alignments and audio synthesis from the generated mel-spectrogram using Griffin-Lim.  Import libraries and setup matplotlib<jupyter_code>import matplotlib
%matplotlib inline
import matplotlib.pylab as plt

import IPython.display as ipd

import sys
sys.path.append('waveglow/')
import numpy as np
import torch

from hparams import create_hparams
from model import Tacotron2
from layers import TacotronSTFT, STFT
from audio_processing import griffin_lim
from train import load_model
from text import text_to_sequence
from denoiser import Denoiser
from unidecode import unidecode
import librosa
def plot_data(data, title=None):
    %matplotlib inline
    if len(data) > 1:
        fig, axes = plt.subplots(len(data), 1, figsize=(int(alignment_graph_width*graph_scale/100), int(alignment_graph_height*graph_scale/100)))
        axes = axes.flatten()
        for i in range(len(data)):
            if title:
                axes[i].set_title(title[i])
            axes[i].imshow(data[i], aspect='auto', origin='bottom', 
                           interpolation='none', cmap='inferno')
        axes[0].set(xlabel="Frames", ylabel="Channels")
        axes[1].set(xlabel="Decoder timestep", ylabel="Encoder timestep")
    else:
        fig, axes = plt.subplots(len(data), 1, figsize=(int(alignment_graph_width*graph_scale/100), int(alignment_graph_height*graph_scale/100)//2))
        if title:
            plt.title(title)
        axes.imshow(data[0], aspect='auto', origin='bottom', interpolation='none', cmap='inferno')
        axes.set(xlabel="Frames", ylabel="Channels")
    fig.canvas.draw()
    plt.show()

#dictionary_path = r"/media/cookie/Samsung PM961/TwiBot/tacotron2/filelists/merged.dict_.txt"
dictionary_path = r"G:\TwiBot\tacotron2\filelists\merged.dict_.txt"
print("Running, Please wait...")
thisdict = {}
for line in reversed((open(dictionary_path, "r").read()).splitlines()):
    thisdict[(line.split(" ", 1))[0]] = (line.split(" ", 1))[1].strip()
print("Dictionary Ready.")
sym = list("☺☻♥♦♣♠•◘○◙♂♀♪♫☼►◄↕‼¶§▬↨↑↓→←∟↔▲▼")
def ARPA(text_, punc=r"!?,.;:␤#-_'\"()[]"):
    text = text_.replace("\n"," "); out = ''
    for word_ in text.split(" "):
        word=word_; end_chars = ''; start_chars = ''
        while any(elem in word for elem in punc) and len(word) > 1:
            if word[-1] in punc: end_chars = word[-1] + end_chars; word = word[:-1]
            elif word[0] in punc: start_chars = start_chars + word[0]; word = word[1:]
            else: break
        try: word_arpa = thisdict[word.upper()]
        except: word_arpa = ''
        if len(word_arpa)!=0: word = "{" + str(word_arpa) + "}"
        out = (out + " " + start_chars + word + end_chars).strip()
    #if out[-1] != "␤": out = out + "␤"
    #if out[0] != "☺": out = "☺" + out
    return out<jupyter_output>Running, Please wait...
Dictionary Ready.<jupyter_text>Setup hparams<jupyter_code>hparams = create_hparams()
hparams.max_decoder_steps = 1000
hparams.gate_threshold = 0.6
hparams.ignore_layers = []
print(str(hparams))
# allows WaveGlow from Ground Truth
from utils import load_wav_to_torch
stft = TacotronSTFT(hparams.filter_length, hparams.hop_length, hparams.win_length,
                    hparams.n_mel_channels, hparams.sampling_rate, hparams.mel_fmin,
                    hparams.mel_fmax)
def load_mel(path):
    audio, sampling_rate, max_value = load_wav_to_torch(path)
    if sampling_rate != stft.sampling_rate:
        raise ValueError("{} {} SR doesn't match target {} SR".format(
            sampling_rate, stft.sampling_rate))
    audio_norm = audio / max(max_value, audio.max(), -audio.min())
    audio_norm = audio_norm.unsqueeze(0)
    audio_norm = torch.autograd.Variable(audio_norm, requires_grad=False)
    melspec = stft.mel_spectrogram(audio_norm)
    return melspec<jupyter_output><empty_output><jupyter_text>Load Tacotron2 model from checkpoint<jupyter_code>checkpoint_path = r"G:\TwiBot\tacotron2-PPP-1.3.0\outdir_truncated1\checkpoint_53000"
print("Loading Tacotron... ", end="")
checkpoint_hparams = torch.load(checkpoint_path)['hparams']
#checkpoint_hparams.parse_json(hparams.to_json())
model = load_model(checkpoint_hparams)
#checkpoint_dict = {k.replace("encoder_speaker_embedding.weight","encoder.encoder_speaker_embedding.weight"): v for k,v in torch.load(checkpoint_path)['state_dict'].items()}
checkpoint_dict = torch.load(checkpoint_path)['state_dict']
model.load_state_dict(checkpoint_dict)
_ = model.cuda().eval().half(); print("Done")

tacotron_speaker_id_lookup = torch.load(checkpoint_path)['speaker_id_lookup']
print("This Tacotron model has been trained for ",torch.load(checkpoint_path)['iteration']," Iterations.")
[x[0] for x in list(model.named_parameters()) if x[0].startswith("decoder")]
print(model)<jupyter_output><empty_output><jupyter_text>Load WaveGlow from Checkpoint<jupyter_code>import torch
import json

waveglow_path = r"H:\TTCheckpoints\waveglow\outdir_EfficientSmallGlobalSpeakerEmbeddings\Testing5\best_val_model" #r"H:\TTCheckpoints\waveglow\outdir_EfficientSmallGlobalSpeakerEmbeddings\Testing2\best_val_model"
config_fpath =  r"H:\TTCheckpoints\waveglow\outdir_EfficientSmallGlobalSpeakerEmbeddings\Testing5\config.json" #r"H:\TTCheckpoints\waveglow\outdir_EfficientSmallGlobalSpeakerEmbeddings\Testing2\config.json"

# Load config file
with open(config_fpath) as f:
    data = f.read()
config = json.loads(data)
train_config = config["train_config"]
global data_config
data_config = config["data_config"]
global dist_config
dist_config = config["dist_config"]
global waveglow_config
waveglow_config = {
    **config["waveglow_config"], 
    'win_length': data_config['win_length'],
    'hop_length': data_config['hop_length']
}
print(waveglow_config)
print(f"Config File from '{config_fpath}' successfully loaded.")

# import the correct model
if waveglow_config["yoyo"]: # efficient_mode # TODO: Add to Config File
    from efficient_model import WaveGlow
else:
    from glow import WaveGlow

# initialize model
print(f"intializing WaveGlow model... ", end="")
waveglow = WaveGlow(**waveglow_config).cuda()
print(f"Done!")

# load checkpoint from file
print(f"loading WaveGlow checkpoint... ", end="")
checkpoint = torch.load(waveglow_path)
waveglow.load_state_dict(checkpoint['model']) # and overwrite initialized weights with checkpointed weights
waveglow.cuda().eval().half() # move to GPU and convert to half precision
print(f"Done!")
#for k in waveglow.convinv:
#    k.float()
print(f"initializing Denoiser... ", end="")
denoiser = Denoiser(waveglow)
print(f"Done!")
waveglow_iters = torch.load(waveglow_path)['iteration']
print(waveglow_iters, "iterations")<jupyter_output>{'yoyo': True, 'yoyo_WN': False, 'n_mel_channels': 160, 'n_flows': 6, 'n_group': 24, 'n_early_every': 6, 'n_early_size': 2, 'memory_efficient': False, 'spect_scaling': False, 'upsample_mode': 'normal', 'WN_config': {'n_layers': 8, 'n_channels': 384, 'kernel_size': 3, 'speaker_embed_dim': 96, 'rezero': False}, 'win_length': 2400, 'hop_length': 600}
Config File from 'H:\TTCheckpoints\waveglow\outdir_EfficientSmallGlobalSpeakerEmbeddings\Testing5\config.json' successfully loaded.
intializing WaveGlow model... Done!
loading WaveGlow checkpoint... Done!
initializing Denoiser... Done!
359500 iterations<jupyter_text>Resave Checkpoint without optimizer<jupyter_code>from os.path import exists
checkpoint['optimizer'] = None

fpath = waveglow_path.replace(r"\best_val_model",r"\best_val_weights")
if  not exists(fpath):
    torch.save(checkpoint, fpath)
    print("Done!")
else:
    print("Already Exists! Skipping!")<jupyter_output>Done!<jupyter_text>(Option 1) Get Speaker ID's from checkpoint<jupyter_code># tacotron_speaker_id_lookup
waveglow_speaker_id_lookup = checkpoint['speaker_lookup']
print(str(waveglow_speaker_id_lookup).replace(",","\n").replace(":"," ->"))<jupyter_output><empty_output><jupyter_text>(Option 2) Rebuild Speaker ID's from training filelist<jupyter_code>#from data_utils import TextMelLoader
#from difflib import get_close_matches
#speaker_ids = TextMelLoader("/media/cookie/Samsung 860 QVO/ClipperDatasetV2/filelists/mel_train_taca2.txt", hparams).speaker_ids
#speaker_ids = TextMelLoader(r"D:\ClipperDatasetV2/filelists/mel_train_taca2.txt", hparams, check_files=False, TBPTT=False).speaker_ids
#print(str(speaker_ids).replace(", ","\n"))<jupyter_output><empty_output><jupyter_text>Load TorchMoji for Style Tokens<jupyter_code># -*- coding: utf-8 -*-

""" Use torchMoji to score texts for emoji distribution.

The resulting emoji ids (0-63) correspond to the mapping
in emoji_overview.png file at the root of the torchMoji repo.

Writes the result to a csv file.
"""
import json
import numpy as np
import os

from torchmoji.sentence_tokenizer import SentenceTokenizer
from torchmoji.model_def import torchmoji_feature_encoding
from torchmoji.global_variables import PRETRAINED_PATH, VOCAB_PATH

print('Tokenizing using dictionary from {}'.format(VOCAB_PATH))
with open(VOCAB_PATH, 'r') as f:
    vocabulary = json.load(f)

maxlen = 180
texts = ["Testing!",]

with torch.no_grad():
    st = SentenceTokenizer(vocabulary, maxlen, ignore_sentences_with_only_custom=True)
    torchmoji = torchmoji_feature_encoding(PRETRAINED_PATH)
    tokenized, _, _ = st.tokenize_sentences(texts) # input array [B] e.g: ["Test?","2nd Sentence!"]
    embedding = torchmoji(tokenized) # returns np array [B, Embed]
    print(embedding)<jupyter_output><empty_output><jupyter_text>Define a measure for Alignment quality in inferred clips.<jupyter_code>def get_mask_from_lengths(lengths, max_len=None):
    if not max_len:
        max_len = torch.max(lengths).long()
    ids = torch.arange(0, max_len, device=lengths.device, dtype=torch.int64)
    mask = (ids < lengths.unsqueeze(1))
    return mask

# New MUCH more performant version, (doesn't support unique padded inputs, just iterate over the batch dim or smthn if you need padded inputs cause this is still way faster)
<EMAIL> # should work and be even faster, but makes it harder to debug and it's already fast enough right now
def alignment_metric(alignments, input_lengths=None, output_lengths=None, average_across_batch=False):
    alignments = alignments.transpose(1,2) # [B, dec, enc] -> [B, enc, dec]
    # alignments [batch size, x, y]
    # input_lengths [batch size] for len_x
    # output_lengths [batch size] for len_y
    if input_lengths == None:
        input_lengths =  torch.ones(alignments.size(0), device=alignments.device)*(alignments.shape[1]-1) # [B] # 147
    if output_lengths == None:
        output_lengths = torch.ones(alignments.size(0), device=alignments.device)*(alignments.shape[2]-1) # [B] # 767
    batch_size = alignments.size(0)
    optimums = torch.sqrt(input_lengths.double().pow(2) + output_lengths.double().pow(2)).view(batch_size)
    
    # [B, enc, dec] -> [B, dec], [B, dec]
    values, cur_idxs = torch.max(alignments, 1) # get max value in column and location of max value
    
    cur_idxs = cur_idxs.float()
    prev_indx = torch.cat((cur_idxs[:,0][:,None], cur_idxs[:,:-1]), dim=1) # shift entire tensor by one.
    dist = ((prev_indx - cur_idxs).pow(2) + 1).pow(0.5) # [B, dec]
    dist.masked_fill_(~get_mask_from_lengths(output_lengths, max_len=dist.size(1)), 0.0) # set dist of padded to zero
    dist = dist.sum(dim=(1)) # get total dist for each B
    diagonalitys = (dist + 1.4142135)/optimums # dist / optimal dist
    
    alignments.masked_fill_(~get_mask_from_lengths(output_lengths, max_len=alignments.size(2))[:,None,:], 0.0)
    att_enc_total = torch.sum(alignments, dim=2)# [B, enc, dec] -> [B, enc]
    
    # calc max (with padding ignored)
    att_enc_total.masked_fill_(~get_mask_from_lengths(input_lengths, max_len=att_enc_total.size(1)), 0.0)
    encoder_max_focus = att_enc_total.max(dim=1)[0] # [B, enc] -> [B]
    
    # calc mean (with padding ignored)
    encoder_avg_focus = att_enc_total.mean(dim=1)   # [B, enc] -> [B]
    encoder_avg_focus *= (att_enc_total.size(1)/input_lengths.float())
    
    # calc min (with padding ignored)
    att_enc_total.masked_fill_(~get_mask_from_lengths(input_lengths, max_len=att_enc_total.size(1)), 1.0)
    encoder_min_focus = att_enc_total.min(dim=1)[0] # [B, enc] -> [B]
    
    # calc average max attention (with padding ignored)
    values.masked_fill_(~get_mask_from_lengths(output_lengths, max_len=values.size(1)), 0.0) # because padding
    avg_prob = values.mean(dim=1)
    avg_prob *= (alignments.size(2)/output_lengths.float()) # because padding
    
    if average_across_batch:
        diagonalitys = diagonalitys.mean()
        encoder_max_focus = encoder_max_focus.mean()
        encoder_min_focus = encoder_min_focus.mean()
        encoder_avg_focus = encoder_avg_focus.mean()
        avg_prob = avg_prob.mean()
    return diagonalitys.cpu(), avg_prob.cpu(), encoder_max_focus.cpu(), encoder_min_focus.cpu(), encoder_avg_focus.cpu()<jupyter_output><empty_output><jupyter_text>Synthesize audio (From Filelist)<jupyter_code>from unidecode import unidecode
import random
from datetime import datetime
import time
from time import sleep
torch.backends.cudnn.enabled = True
torch.backends.cudnn.benchmark = False

"""
|(Game) Them's Fightin' Herds_Oleander|150
|(Game) Them's Fightin' Herds_Fred|151
|(Game) Them's Fightin' Herds_Pom|152
|(Game) Them's Fightin' Herds_Velvet|153
|(Game) Them's Fightin' Herds_Arizona|154
|(Game) Them's Fightin' Herds_Tianhuo|155
|(Game) Elite Dangerous_Eli|156
|(Audiobook) A Little Bit Wicked_Skystar|157
|(Audiobook) Dr. Who_Doctor|158
|(Show) My Little Pony_Applejack|159
|(Show) My Little Pony_Rainbow|160
|(Show) My Little Pony_Pinkie|161
|(Show) My Little Pony_Rarity|162
|(Show) My Little Pony_Spike|163
|(Show) My Little Pony_Fluttershy|164
|(Show) My Little Pony_Nightmare Moon|165
|(Show) Dan Vs_Dan|166
|(Show) My Little Pony_Twilight|167
|(Show) My Little Pony_Scootaloo|168
|(Show) My Little Pony_Big Mac|169
|(Show) My Little Pony_Sweetie Belle|170
|(Show) My Little Pony_Apple Bloom|171
"""

speakers = """
|(Show) My Little Pony_Twilight|167
""".replace("_",", ").replace("("," ").replace(")",", ").split("\n")[1:-1]

narrators = """
|(Audiodrama) <NAME>|1
|(Game) Them's Fightin' Herds_Oleander|150
|(Game) Them's Fightin' Herds_Pom|152
|(Game) Them's Fightin' Herds_Velvet|153
|(Game) Them's Fightin' Herds_Arizona|154
|(Game) Them's Fightin' Herds_Tianhuo|155
|(Show) My Little Pony_Applejack|159
|(Show) My Little Pony_Rainbow|160
|(Show) My Little Pony_Pinkie|161
|(Show) My Little Pony_Rarity|162
|(Show) My Little Pony_Spike|163
|(Show) My Little Pony_Fluttershy|164
|(Show) Dan Vs_Dan|166
|(Show) My Little Pony_Twilight|167
|(Show) My Little Pony_Scootaloo|168
|(Show) My Little Pony_Sweetie Belle|170
|(Show) My Little Pony_Apple Bloom|171
""".replace("_",", ").replace("("," ").replace(")",", ").split("\n")[1:-1]

texts = """
Mmmmmmmmmmmmmmmmmmmmmmmm, that feels nice.
Mmmmmmmmmmmmmmmmmmm, that feels nice.
Mmmmmmmmmmmmmmm, that feels nice.
Mmmmmmmmmmmm that feels nice.
Mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm that feels nice.
Mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm that feels nice.
Mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm that feels nice.
Mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm that feels nice.
"""
_audio_path_override = None
_speaker_id_override = None
style_mode = 'torchmoji_hidden' # Options = 'mel','token','zeros','torchmoji_hidden','torchmoji_string'

acceptable_score = 0.8 # sufficient score to just skip ahead instead of checking/generating more outputs
absolutely_required_score = 0.3 # retry forever until this score is reached
absolute_maximum_tries = 256 # this is per text text input

# Score Parameters
diagonality_weighting = 0.5 # 'stutter factor', a penalty for clips where the model jumps back and forwards in the sentence.
max_focus_weighting = 1.0   # 'stuck factor', a penalty for clips that spend execisve time on the same letter.
min_focus_weighting = 1.0   # 'miniskip factor', a penalty for skipping/ignoring small parts of the input text.
avg_focus_weighting = 1.0   # 'skip factor', a penalty for skipping very large parts of the input text
max_attempts = 128 # retries at each clip # this is per text input
batch_size_per_text = 128 # minibatch_size per unique text input
simultaneous_texts = 1 # num unique text inputs per batch

max_decoder_steps = 1600
max_text_segment_length = 120
gate_threshold = 0.7
gate_delay = 3
use_arpabet = 1

sigma = 0.95
audio_save_path = r"D:\Downloads\infer\testing"
output_filename = 'TestingOutput'
save_wavs = 1 # saves wavs to infer folder

show_all_attempt_scores = 0
show_audio_overwrite_warnings = 1
show_input_text = 1
show_best_score = 1
show_audio  = 1
show_graphs_tacotron = 1
show_graphs_waveglow = 1
status_updates = 1 # ... Done
time_to_gen = 1
graph_scale = 0.5
alignment_graph_width = 3840
alignment_graph_height = 1920

model.decoder.gate_delay = gate_delay
model.decoder.max_decoder_steps = max_decoder_steps
model.decoder.gate_threshold = gate_threshold

os.makedirs(audio_save_path, exist_ok=True)
texts_segmented = [x for x in texts.split("\n") if len(x.strip())]
total_len = len(texts_segmented)

continue_from = 0 # skip
counter = 0
text_batch_in_progress = []
for text_index, text in enumerate(texts_segmented):
    if text_index < continue_from: print(f"Skipping {text_index}.\t",end=""); counter+=1; continue
    print(f"{text_index}/{total_len}|{datetime.now()}")
    
    # setup the text batches
    text_batch_in_progress.append(text)
    if (len(text_batch_in_progress) == simultaneous_texts) or (text_index == (len(texts_segmented)-1)): # if text batch ready or final input
        text_batch = text_batch_in_progress
        text_batch_in_progress = []
        if (text_index == (len(texts_segmented)-1)): # if final text input
            simultaneous_texts = len(text_batch) # ensure batch size is still correct
    else:
        continue # if batch not ready, add another text
    
    # pick the speakers for the texts
    speaker_ids = [random.choice(speakers).split("|")[2] if ('"' in text) else random.choice(narrators).split("|")[2] for text in text_batch] # pick speaker if quotemark in text, else narrator
    text_batch  = [text.replace('"',"") for text in text_batch] # remove quotes from text
    
    if _audio_path_override != None:
        audio_path = _audio_path_override
    if _speaker_id_override != None:
        speaker_id = _speaker_id_override
    
    # get speaker_ids (tacotron)
    tacotron_speaker_ids = [tacotron_speaker_id_lookup[int(speaker_id)] for speaker_id in speaker_ids]
    tacotron_speaker_ids = torch.LongTensor(tacotron_speaker_ids).cuda().repeat_interleave(batch_size_per_text)
    
    # get speaker_ids (waveglow)
    waveglow_speaker_ids = [waveglow_speaker_id_lookup[int(speaker_id)] for speaker_id in speaker_ids]
    waveglow_speaker_ids = torch.LongTensor(waveglow_speaker_ids).cuda()
    
    # style
    if style_mode == 'mel':
        mel = load_mel(audio_path.replace(".npy",".wav")).cuda().half()
        style_input = mel
    elif style_mode == 'token':
        pass
        #style_input =
    elif style_mode == 'zeros':
        style_input = None
    elif style_mode == 'torchmoji_hidden':
        try:
            tokenized, _, _ = st.tokenize_sentences(text_batch) # input array [B] e.g: ["Test?","2nd Sentence!"]
        except:
            raise Exception(f"text\n{text_batch}\nfailed to tokenize.")
        try:
            embedding = torchmoji(tokenized) # returns np array [B, Embed]
        except Exception as ex:
            print(f'Exception: {ex}')
            print(f"text: {text_batch} failed to process.")
            #raise Exception(f"text\n{text}\nfailed to process.")
        style_input = torch.from_numpy(embedding).cuda().half().repeat_interleave(batch_size_per_text, dim=0)
    elif style_mode == 'torchmoji_string':
        style_input = text_batch
        raise NotImplementedError
    else:
        raise NotImplementedError
    
    # check punctuation
    valid_last_char = '-,.?!;:' # valid final characters in texts
    text_batch = [text+'.' if (text[-1] not in ',.?!;:') else text for text in text_batch]
    
    # parse text
    text_batch = [unidecode(text.replace("...",". ").replace("  "," ").strip()) for text in text_batch] # remove eclipses, double spaces, unicode and spaces before/after the text.
    if show_input_text: # debug
        print("raw_text:\n", "\n".join([str(j)+': \''+text+'\'' for j, text in enumerate(text_batch)]), sep='')
    if use_arpabet: # convert texts to ARPAbet (phonetic) versions.
        text_batch = [ARPA(text) for text in text_batch]
    if show_input_text: # debug
        print("model_input:\n", "\n".join([str(j)+': \''+text+'\'' for j, text in enumerate(text_batch)]), sep='')
    
    with torch.no_grad():
        if time_to_gen:
            start_time = time.time()
        
        # convert texts to sequence, pad where appropriate and move to GPU
        sequence_split = [torch.LongTensor(text_to_sequence(text, ['english_cleaners'])) for text in text_batch] # convert texts to numpy representation
        text_lengths = torch.tensor([seq.size(0) for seq in sequence_split])
        max_len = text_lengths.max().item()
        sequence = torch.zeros(text_lengths.size(0), max_len).long() # create large tensor to move each text input into
        for i in range(text_lengths.size(0)): # move each text into padded input tensor
            sequence[i, :sequence_split[i].size(0)] = sequence_split[i]
        sequence = sequence.cuda().long().repeat_interleave(batch_size_per_text, dim=0) # move to GPU and repeat text
        text_lengths = text_lengths.cuda().long() # move to GPU
        #print("max_len =", max_len) # debug
        #print( get_mask_from_lengths(text_lengths).shape ) # debug
        #sequence = torch.autograd.Variable(torch.from_numpy(sequence)).cuda().long().repeat_interleave(batch_size_per_text, 0)# convert numpy to tensor and repeat for each text
        
        # debug
        text_lengths = text_lengths.clone()
        sequence = sequence.clone()
        
        for i in range(1):
            try:
                best_score = np.ones(simultaneous_texts) * -9e9
                tries      = np.zeros(simultaneous_texts)
                best_generations = [0]*simultaneous_texts
                best_score_str = ['']*simultaneous_texts
                while np.amin(best_score) < acceptable_score:
                    # run inference
                    if status_updates: print("Running Tacotron2... ", end='')
                    mel_batch_outputs, mel_batch_outputs_postnet, gate_batch_outputs, alignments_batch = model.inference(sequence, tacotron_speaker_ids, style_input=style_input, style_mode=style_mode, text_lengths=text_lengths.repeat_interleave(batch_size_per_text, dim=0))
                    
                    # find metrics for each item
                    gate_batch_outputs[:,:20] = 0 # ignore gate predictions for the first bit
                    output_lengths = gate_batch_outputs.argmax(dim=1)+gate_delay
                    diagonality_batch, avg_prob_batch, enc_max_focus_batch, enc_min_focus_batch, enc_avg_focus_batch = alignment_metric(alignments_batch, input_lengths=text_lengths.repeat_interleave(batch_size_per_text, dim=0), output_lengths=output_lengths)
                    
                    batch = list(zip(
                        mel_batch_outputs.split(1,dim=0),
                        mel_batch_outputs_postnet.split(1,dim=0),
                        gate_batch_outputs.split(1,dim=0),
                        alignments_batch.split(1,dim=0),
                        diagonality_batch,
                        avg_prob_batch,
                        enc_max_focus_batch,
                        enc_min_focus_batch,
                        enc_avg_focus_batch,))
                    # split batch into items
                    
                    for j in range(simultaneous_texts): # process each set of text spectrograms seperately
                        start, end = (j*batch_size_per_text), ((j+1)*batch_size_per_text)
                        sametext_batch = batch[start:end] # seperate the full batch into pieces that use the same input text
                        
                        # process all items related to the j'th text input
                        for k, (mel_outputs, mel_outputs_postnet, gate_outputs, alignments, diagonality, avg_prob, enc_max_focus, enc_min_focus, enc_avg_focus) in enumerate(sametext_batch):
                            # factors that make up score
                            weighted_score =  avg_prob.item() # general alignment quality
                            weighted_score -= (max(diagonality.item(),1.11)-1.11) * diagonality_weighting  # consistent pace
                            weighted_score -= max((enc_max_focus.item()-20), 0) * 0.005 * max_focus_weighting # getting stuck on pauses/phones
                            weighted_score -= max(0.9-enc_min_focus.item(),0) * min_focus_weighting # skipping single enc outputs
                            weighted_score -= max(2.5-enc_avg_focus.item(), 0) * avg_focus_weighting # skipping most enc outputs
                            score_str = f"{round(diagonality.item(),3)} {round(avg_prob.item()*100,2)}% {round(weighted_score,4)} {round(max((enc_max_focus.item()-20), 0) * 0.005 * max_focus_weighting,2)} {round(max(0.9-enc_min_focus.item(),0),2)}|"
                            if weighted_score > best_score[j]:
                                best_score[j] = weighted_score
                                best_score_str[j] = score_str
                                best_generations[j] = [mel_outputs, mel_outputs_postnet, gate_outputs, alignments]
                            if show_all_attempt_scores:
                                print(score_str, end="")
                            tries[j]+=1
                            if np.amin(tries) >= max_attempts and np.amin(best_score) > (absolutely_required_score-1):
                                raise StopIteration
                            if np.amin(tries) >= absolute_maximum_tries:
                                print(f"Absolutely required score not achieved in {absolute_maximum_tries} attempts - ", end='')
                                raise StopIteration
                    
                    if np.amin(tries) < (max_attempts-1):
                        print('Acceptable alignment/diagonality not reached. Retrying.')
                    elif np.amin(best_score) < absolutely_required_score:
                        print('Score less than absolutely required score. Retrying extra.')
            except StopIteration:
                del batch
                if status_updates: print("Done")
                pass
            # [[mel, melpost, gate, align], [mel, melpost, gate, align], [mel, melpost, gate, align]] -> [[mel, mel, mel], [melpost, melpost, melpost], [gate, gate, gate], [align, align, align]]
            
            # zip is being weird so alternative used
            mel_batch_outputs, mel_batch_outputs_postnet, gate_batch_outputs, alignments_batch = [x[0][0].T for x in best_generations], [x[1][0].T for x in best_generations], [x[2][0] for x in best_generations], [x[3][0] for x in best_generations] # pickup whatever was the best attempts
            
            # stack arrays into tensors
            gate_batch_outputs = torch.nn.utils.rnn.pad_sequence(gate_batch_outputs, batch_first=True, padding_value=0)
#            print(gate_batch_outputs.shape)
            max_length = torch.max(gate_batch_outputs.argmax(dim=1)) # get highest duration
            mel_batch_outputs = torch.nn.utils.rnn.pad_sequence(mel_batch_outputs, batch_first=True, padding_value=-11.6).transpose(1,2)[:,:,:max_length]
            mel_batch_outputs_postnet = torch.nn.utils.rnn.pad_sequence(mel_batch_outputs_postnet, batch_first=True, padding_value=-11.6).transpose(1,2)[:,:,:max_length]
            alignments_batch = torch.nn.utils.rnn.pad_sequence(alignments_batch, batch_first=True, padding_value=0)[:,:max_length,:]
            
            if status_updates: print("Running WaveGlow... ", end='')
            audio_batch = waveglow.infer(mel_batch_outputs_postnet, speaker_ids=waveglow_speaker_ids, sigma=sigma)
            audio_denoised_batch = denoiser(audio_batch, strength=0.0001)[:, 0]
            if status_updates: print('Done')
            
            audio_len = 0
            for j, (audio, audio_denoised) in enumerate(zip(audio_batch.split(1, dim=0), audio_denoised_batch.split(1, dim=0))):
                # remove WaveGlow padding
                audio_end = (gate_batch_outputs[j].argmax()+gate_delay) * hparams.hop_length
                audio = audio[:,:audio_end]
                audio_denoised = audio_denoised[:,:audio_end]
                
                # remove Tacotron2 padding
                spec_end = gate_batch_outputs[j].argmax()+gate_delay
                mel_outputs = mel_batch_outputs.split(1, dim=0)[j][:,:,:spec_end]
                ##print("mel_outputs.split(blah)[j].shape", mel_outputs.shape)
                mel_outputs_postnet = mel_batch_outputs_postnet.split(1, dim=0)[j][:,:,:spec_end]
                ##print("alignments_batch.shape", alignments_batch.shape)
                alignments = alignments_batch.split(1, dim=0)[j][:,:spec_end,:text_lengths[j]]
                ##print("alignments.split(blah)[j].shape", alignments.shape)
                
                if show_best_score:
                    print(f"Score: {round(best_score[j],3)}\t\tStats: {best_score_str[j]}  Verified: {[x.item() for x in alignment_metric(alignments)]}")
                if show_graphs_tacotron:
                    plot_data((mel_outputs_postnet.float().data.cpu().numpy()[0],
                           alignments.float().data.cpu().numpy()[0].T), title=["Spectrogram (Tacotron)","Alignment (Tacotron)"])
                if show_audio:
                        ipd.display(ipd.Audio(audio.cpu().numpy(), rate=hparams.sampling_rate))        
                if save_wavs:
                    save_audio_path = os.path.join(audio_save_path,f"audio_{counter//300:02}_{counter:05}.wav")
                    if os.path.exists(save_audio_path):
                        if show_audio_overwrite_warnings:
                            print(f"File already found at [{save_audio_path}], overwriting.")
                        os.remove(save_audio_path)
                    if status_updates: print(f"Saving clip to [{save_audio_path}]... ", end="")
                    librosa.output.write_wav(save_audio_path, np.swapaxes(audio.float().cpu().numpy(),0,1), hparams.sampling_rate)
                    if status_updates: print("Done")
                if show_graphs_waveglow and save_wavs:
                    plot_data([load_mel(save_audio_path).float().data.cpu().numpy()[0],], title = "Spectrogram (WaveGlow)")
                counter+=1
                audio_len+=audio_end
            
            if time_to_gen:
                audio_seconds_generated = round(audio_len.item()/hparams.sampling_rate,3)
                print(f"Took {round(time.time()-start_time,3)}s to generate {audio_seconds_generated}s of audio. (best of {tries.sum().astype('int')} tries)")
            
            print("")
_text = None; _audio_path = None; _speaker_id = None

# Merge clips and output the concatenated result
from glob import glob
import os

# get number of intermediate concatenations required (Sox can only merge 340~ files at a time)
n_audio_batches = -(-len( glob(os.path.join(audio_save_path, "audio_*_*.wav")) ) // 300)

# ensure path ends in .wav
if not output_filename[-4:].lower() == '.wav':
    output_filename+='.wav'

for i in range(n_audio_batches):
    print(f"Merging audio files {i*300} to {((i+1)*300)-1}... ", end='')
    os.system(f'sox {os.path.join(audio_save_path, f"audio_{i:02}_*.wav")} -b 16 {os.path.join(audio_save_path, f"concat_{i:02}.wav")}')
    print("Done")
print(f"Saving output to '{os.path.join(audio_save_path, output_filename)}'... ", end='')
os.system(f'sox "{os.path.join(audio_save_path, "concat_*.wav")}" -b 16 "{os.path.join(audio_save_path, output_filename)}"') # merge the merged files into a final output. bit depth of 16 required to go over 4 hour length
print("Done.")<jupyter_output><empty_output><jupyter_text>Synthesize audio (From Text List e.g Fimfic)<jupyter_code>def chunks(lst, n):
    """Yield successive n-sized chunks from lst."""
    # list(chunks([0,1,2,3,4,5,6,7,8,9],2)) -> [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9]]
    for i in range(0, len(lst), n):
        yield lst[i:i + n]

def parse_txt_into_quotes(fpath):
    texts = open(fpath, "r", encoding="utf-8").read()
    
    quo ='"'; texts = [f'"{text.replace(quo,"").strip()}"' if i%2 else text.replace(quo,"").strip() for i, text in enumerate(unidecode(texts).split('"'))]
    
    texts_segmented = []
    for text in texts:
        text = text.strip()
        if not len(text.replace('"','').strip()): continue
        text = text\
            .replace("\n"," ")\
            .replace("  "," ")\
            .replace("> --------------------------------------------------------------------------","")
        if len(text) > max_text_segment_length:
            for seg in [x.strip() for x in text.split(".") if len(x.strip()) if x is not '"']: 
                if '"' in text:
                    if seg[0] != '"': seg='"'+seg
                    if seg[-1] != '"': seg+='"'
                texts_segmented.append(seg)
        else:
            texts_segmented.append(text.strip())
    return texts_segmented
from unidecode import unidecode
import random
from datetime import datetime
import time
from time import sleep
torch.backends.cudnn.enabled = True
torch.backends.cudnn.benchmark = False

"""
|(Game) Them's Fightin' Herds_Oleander|150
|(Game) Them's Fightin' Herds_Fred|151
|(Game) Them's Fightin' Herds_Pom|152
|(Game) Them's Fightin' Herds_Velvet|153
|(Game) Them's Fightin' Herds_Arizona|154
|(Game) Them's Fightin' Herds_Tianhuo|155
|(Game) Elite Dangerous_Eli|156
|(Audiobook) A Little Bit Wicked_Skystar|157
|(Audiobook) Dr. Who_Doctor|158
|(Show) My Little Pony_Applejack|159
|(Show) My Little Pony_Rainbow|160
|(Show) My Little Pony_Pinkie|161
|(Show) My Little Pony_Rarity|162
|(Show) My Little Pony_Spike|163
|(Show) My Little Pony_Fluttershy|164
|(Show) My Little Pony_Nightmare Moon|165
|(Show) Dan Vs_Dan|166
|(Show) My Little Pony_Twilight|167
|(Show) My Little Pony_Scootaloo|168
|(Show) My Little Pony_Big Mac|169
|(Show) My Little Pony_Sweetie Belle|170
|(Show) My Little Pony_Apple Bloom|171
"""

speakers = """
|(Show) My Little Pony_Twilight|167
""".replace("_",", ").replace("("," ").replace(")",", ").split("\n")[1:-1]

narrators = """
|(Show) My Little Pony_Applejack|159
""".replace("_",", ").replace("("," ").replace(")",", ").split("\n")[1:-1]

_audio_path_override = None
_speaker_id_override = None
style_mode = 'torchmoji_hidden' # Options = 'mel','token','zeros','torchmoji_hidden','torchmoji_string'

acceptable_score = 0.8 # sufficient score to just skip ahead instead of checking/generating more outputs
absolutely_required_score = 0.3 # retry forever until this score is reached
absolute_maximum_tries = 512 # this is per text text input

# Score Parameters
diagonality_weighting = 0.5 # 'stutter factor', a penalty for clips where the model jumps back and forwards in the sentence.
max_focus_weighting = 1.0   # 'stuck factor', a penalty for clips that spend execisve time on the same letter.
min_focus_weighting = 1.0   # 'miniskip factor', a penalty for skipping/ignoring small parts of the input text.
avg_focus_weighting = 1.0   # 'skip factor', a penalty for skipping very large parts of the input text
max_attempts = 256 # retries at each clip # this is per text input
batch_size_per_text = 256 # minibatch_size per unique text input
simultaneous_texts = 1 # num unique text inputs per batch

max_decoder_steps = 1600
max_text_segment_length = 120
gate_threshold = 0.7
gate_delay = 3
use_arpabet = 1

sigma = 0.95
audio_save_path = r"D:\Downloads\infer\audio"
output_filename = 'Mort Takes a Holiday'
save_wavs = 1 # saves wavs to infer folder

show_all_attempt_scores = 0
show_audio_overwrite_warnings = 1
show_input_text = 1
show_best_score = 1
show_audio  = 1
show_graphs = 1
status_updates = 1 # ... Done
time_to_gen = 1
graph_scale = 0.5
alignment_graph_width = 3840
alignment_graph_height = 1920

model.decoder.gate_delay = gate_delay
model.decoder.max_decoder_steps = max_decoder_steps
model.decoder.gate_threshold = gate_threshold

file_path = r"D:\Downloads\infer\text\Mort Takes a Holiday.txt"

texts_segmented = parse_txt_into_quotes(file_path)

total_len = len(texts_segmented)

# 0 init
# 1 append
# 2 append, generate, blank
# 1 append
# 2 append, generate, blank
# 1
# 2

continue_from = 2545 # skip
counter = 0
text_batch_in_progress = []
for text_index, text in enumerate(texts_segmented):
    if text_index < continue_from: print(f"Skipping {text_index}.\t",end=""); counter+=1; continue
    print(f"{text_index}/{total_len}|{datetime.now()}")
    
    # setup the text batches
    text_batch_in_progress.append(text)
    if (len(text_batch_in_progress) == simultaneous_texts) or (text_index == (len(texts_segmented)-1)): # if text batch ready or final input
        text_batch = text_batch_in_progress
        text_batch_in_progress = []
    else:
        continue # if batch not ready, add another text
    
    # pick the speakers for the texts
    speaker_ids = [random.choice(speakers).split("|")[2] if ('"' in text) else random.choice(narrators).split("|")[2] for text in text_batch] # pick speaker if quotemark in text, else narrator
    text_batch  = [text.replace('"',"") for text in text_batch] # remove quotes from text
    
    if _audio_path_override != None:
        audio_path = _audio_path_override
    if _speaker_id_override != None:
        speaker_id = _speaker_id_override
    
    # get speaker_ids (tacotron)
    tacotron_speaker_ids = [tacotron_speaker_id_lookup[int(speaker_id)] for speaker_id in speaker_ids]
    tacotron_speaker_ids = torch.LongTensor(tacotron_speaker_ids).cuda().repeat_interleave(batch_size_per_text)
    
    # get speaker_ids (waveglow)
    waveglow_speaker_ids = [waveglow_speaker_id_lookup[int(speaker_id)] for speaker_id in speaker_ids]
    waveglow_speaker_ids = torch.LongTensor(waveglow_speaker_ids).cuda()
    
    # style
    if style_mode == 'mel':
        mel = load_mel(audio_path.replace(".npy",".wav")).cuda().half()
        style_input = mel
    elif style_mode == 'token':
        pass
        #style_input =
    elif style_mode == 'zeros':
        style_input = None
    elif style_mode == 'torchmoji_hidden':
        try:
            tokenized, _, _ = st.tokenize_sentences(text_batch) # input array [B] e.g: ["Test?","2nd Sentence!"]
        except:
            raise Exception(f"text\n{text_batch}\nfailed to tokenize.")
        try:
            embedding = torchmoji(tokenized) # returns np array [B, Embed]
        except Exception as ex:
            print(f'Exception: {ex}')
            print(f"text: '{text_batch}' failed to process.")
            #raise Exception(f"text\n{text}\nfailed to process.")
        style_input = torch.from_numpy(embedding).cuda().half().repeat_interleave(batch_size_per_text, dim=0)
    elif style_mode == 'torchmoji_string':
        style_input = text_batch
        raise NotImplementedError
    else:
        raise NotImplementedError
    
    # check punctuation
    valid_last_char = '-,.?!;:' # valid final characters in texts
    text_batch = [text+'.' if (text[-1] not in ',.?!;:') else text for text in text_batch]
    
    # parse text
    text_batch = [unidecode(text.replace("...",". ").replace("  "," ").strip()) for text in text_batch] # remove eclipses, double spaces, unicode and spaces before/after the text.
    if show_input_text: # debug
        print("raw_text:\n", "\n".join([str(j)+': \''+text+'\'' for j, text in enumerate(text_batch)]), sep='')
    if use_arpabet: # convert texts to ARPAbet (phonetic) versions.
        text_batch = [ARPA(text) for text in text_batch]
    if show_input_text: # debug
        print("model_input:\n", "\n".join([str(j)+': \''+text+'\'' for j, text in enumerate(text_batch)]), sep='')
    
    with torch.no_grad():
        if time_to_gen:
            start_time = time.time()
        
        # convert texts to sequence, pad where appropriate and move to GPU
        sequence_split = [torch.LongTensor(text_to_sequence(text, ['english_cleaners'])) for text in text_batch] # convert texts to numpy representation
        text_lengths = torch.tensor([seq.size(0) for seq in sequence_split])
        max_len = text_lengths.max().item()
        sequence = torch.zeros(text_lengths.size(0), max_len).long() # create large tensor to move each text input into
        for i in range(text_lengths.size(0)): # move each text into padded input tensor
            sequence[i, :sequence_split[i].size(0)] = sequence_split[i]
        sequence = sequence.cuda().long().repeat_interleave(batch_size_per_text, dim=0) # move to GPU and repeat text
        text_lengths = text_lengths.cuda().long() # move to GPU
        #print("max_len =", max_len) # debug
        #print( get_mask_from_lengths(text_lengths).shape ) # debug
        #sequence = torch.autograd.Variable(torch.from_numpy(sequence)).cuda().long().repeat_interleave(batch_size_per_text, 0)# convert numpy to tensor and repeat for each text
        
        # debug
        text_lengths = text_lengths.clone()
        sequence = sequence.clone()
        
        for i in range(1):
            try:
                best_score = np.ones(simultaneous_texts) * -9e9
                tries      = np.zeros(simultaneous_texts)
                best_generations = [0]*simultaneous_texts
                best_score_str = ['']*simultaneous_texts
                while np.amin(best_score) < acceptable_score:
                    # run inference
                    if status_updates: print("Running Tacotron2... ", end='')
                    mel_batch_outputs, mel_batch_outputs_postnet, gate_batch_outputs, alignments_batch = model.inference(sequence, tacotron_speaker_ids, style_input=style_input, style_mode=style_mode, text_lengths=text_lengths.repeat_interleave(batch_size_per_text, dim=0))
                    
                    # find metrics for each item
                    gate_batch_outputs[:,:20] = 0 # ignore gate predictions for the first 0.05s
                    output_lengths = gate_batch_outputs.argmax(dim=1)+gate_delay
                    diagonality_batch, avg_prob_batch, enc_max_focus_batch, enc_min_focus_batch, enc_avg_focus_batch = alignment_metric(alignments_batch, input_lengths=text_lengths.repeat_interleave(batch_size_per_text, dim=0), output_lengths=output_lengths)
                    
                    batch = list(zip(
                        mel_batch_outputs.split(1,dim=0),
                        mel_batch_outputs_postnet.split(1,dim=0),
                        gate_batch_outputs.split(1,dim=0),
                        alignments_batch.split(1,dim=0),
                        diagonality_batch,
                        avg_prob_batch,
                        enc_max_focus_batch,
                        enc_min_focus_batch,
                        enc_avg_focus_batch,))
                    # split batch into items
                    
                    for j in range(simultaneous_texts): # process each set of text spectrograms seperately
                        start, end = (j*batch_size_per_text), ((j+1)*batch_size_per_text)
                        sametext_batch = batch[start:end] # seperate the full batch into pieces that use the same input text
                        
                        # process all items related to the j'th text input
                        for k, (mel_outputs, mel_outputs_postnet, gate_outputs, alignments, diagonality, avg_prob, enc_max_focus, enc_min_focus, enc_avg_focus) in enumerate(sametext_batch):
                            # factors that make up score
                            weighted_score =  avg_prob.item() # general alignment quality
                            weighted_score -= (max(diagonality.item(),1.11)-1.11) * diagonality_weighting  # consistent pace
                            weighted_score -= max((enc_max_focus.item()-20), 0) * 0.005 * max_focus_weighting # getting stuck on pauses/phones
                            weighted_score -= max(0.9-enc_min_focus.item(),0) * min_focus_weighting # skipping single enc outputs
                            weighted_score -= max(2.5-enc_avg_focus.item(), 0) * avg_focus_weighting # skipping most enc outputs
                            score_str = f"{round(diagonality.item(),3)} {round(avg_prob.item()*100,2)}% {round(weighted_score,4)} {round(max((enc_max_focus.item()-20), 0) * 0.005 * max_focus_weighting,2)} {round(max(0.9-enc_min_focus.item(),0),2)}|"
                            if weighted_score > best_score[j]:
                                best_score[j] = weighted_score
                                best_score_str[j] = score_str
                                best_generations[j] = [mel_outputs, mel_outputs_postnet, gate_outputs, alignments]
                            if show_all_attempt_scores:
                                print(score_str, end="")
                            tries[j]+=1
                            if np.amin(tries) >= max_attempts and np.amin(best_score) > (absolutely_required_score-1):
                                raise StopIteration
                            if np.amin(tries) >= absolute_maximum_tries:
                                print(f"Absolutely required score not achieved in {absolute_maximum_tries} attempts - ", end='')
                                raise StopIteration
                    
                    if np.amin(tries) < (max_attempts-1):
                        print('Acceptable alignment/diagonality not reached. Retrying.')
                    elif np.amin(best_score) < absolutely_required_score:
                        print('Score less than absolutely required score. Retrying extra.')
            except StopIteration:
                del batch
                if status_updates: print("Done")
                pass
            # [[mel, melpost, gate, align], [mel, melpost, gate, align], [mel, melpost, gate, align]] -> [[mel, mel, mel], [melpost, melpost, melpost], [gate, gate, gate], [align, align, align]]
            
            # zip is being weird so alternative used
            mel_batch_outputs, mel_batch_outputs_postnet, gate_batch_outputs, alignments_batch = [x[0] for x in best_generations], [x[1] for x in best_generations], [x[2] for x in best_generations], [x[3] for x in best_generations] # pickup whatever was the best attempts
            
            # stack arrays into tensors
            gate_batch_outputs = torch.cat(gate_batch_outputs, dim=0)
            max_length = torch.max(gate_batch_outputs.argmax(dim=1)) # get highest duration
            mel_batch_outputs = torch.cat(mel_batch_outputs, dim=0)[:,:,:max_length]
            mel_batch_outputs_postnet = torch.cat(mel_batch_outputs_postnet, dim=0)[:,:,:max_length]
            alignments_batch = torch.cat(alignments_batch, dim=0)[:,:max_length,:]
            ##print("max_length =", max_length)
            ##print("gate_batch_outputs.argmax(dim=1) =", gate_batch_outputs.argmax(dim=1))
            ##print("mel_batch_outputs.shape", mel_batch_outputs.shape)
            ##print("mel_batch_outputs_postnet.shape", mel_batch_outputs_postnet.shape)
            ##print("alignments_batch.shape", alignments_batch.shape)
            
            if status_updates: print("Running WaveGlow... ", end='')
            audio_batch = waveglow.infer(mel_batch_outputs_postnet, speaker_ids=waveglow_speaker_ids, sigma=sigma)
            audio_denoised_batch = denoiser(audio_batch, strength=0.0001)[:, 0]
            if status_updates: print('Done')
            
            audio_len = 0
            for j, (audio, audio_denoised) in enumerate(zip(audio_batch.split(1, dim=0), audio_denoised_batch.split(1, dim=0))):
                # remove WaveGlow padding
                audio_end = (gate_batch_outputs[j].argmax()+gate_delay) * hparams.hop_length
                audio = audio[:,:audio_end]
                audio_denoised = audio_denoised[:,:audio_end]
                
                # remove Tacotron2 padding
                spec_end = gate_batch_outputs[j].argmax()+gate_delay
                mel_outputs = mel_batch_outputs.split(1, dim=0)[j][:,:,:spec_end]
                ##print("mel_outputs.split(blah)[j].shape", mel_outputs.shape)
                mel_outputs_postnet = mel_batch_outputs_postnet.split(1, dim=0)[j][:,:,:spec_end]
                ##print("alignments_batch.shape", alignments_batch.shape)
                alignments = alignments_batch.split(1, dim=0)[j][:,:spec_end,:text_lengths[j]]
                ##print("alignments.split(blah)[j].shape", alignments.shape)
                
                if show_best_score:
                    print(f"Score: {round(best_score[j],3)}\t\tStats: {best_score_str[j]}  Verified: {[x.item() for x in alignment_metric(alignments)]}")
                if show_graphs:
                    plot_data((mel_outputs_postnet.float().data.cpu().numpy()[0],
                           alignments.float().data.cpu().numpy()[0].T))
                if show_audio:
                        ipd.display(ipd.Audio(audio.cpu().numpy(), rate=hparams.sampling_rate))        
                if save_wavs:
                    save_audio_path = os.path.join(audio_save_path,f"audio_{counter//300:02}_{counter:05}.wav")
                    if os.path.exists(save_audio_path):
                        if show_audio_overwrite_warnings:
                            print(f"File already found at [{save_audio_path}], overwriting.")
                        os.remove(save_audio_path)
                    if status_updates: print(f"Saving clip to [{save_audio_path}]... ", end="")
                    librosa.output.write_wav(save_audio_path, np.swapaxes(audio.float().cpu().numpy(),0,1), hparams.sampling_rate)
                    if status_updates: print("Done")
                counter+=1
                audio_len+=audio_end
            
            if time_to_gen:
                audio_seconds_generated = round(audio_len.item()/hparams.sampling_rate,3)
                print(f"Took {round(time.time()-start_time,3)}s to generate {audio_seconds_generated}s of audio. (best of {tries.sum().astype('int')} tries)")
                #print("spec_end/max_len = ", spec_end/max_len) # debug
                #print("spec_end/max_len = ", spec_end/max_len) # debug
            
            print("")
_text = None; _audio_path = None; _speaker_id = None

# Merge clips and output the concatenated result
from glob import glob
import os

# get number of intermediate concatenations required (Sox and only merge 340~ files at a time)
n_audio_batches = round(len( glob(os.path.join(audio_save_path, "audio_*_*.wav")) ) / 300)

# ensure path ends in .wav
if not output_filename[-4:].lower() == '.wav':
    output_filename+='.wav'

for i in range(n_audio_batches):
    print(f"Merging audio files {i*300} to {((i+1)*300)-1}... ", end='')
    os.system(f'sox {os.path.join(audio_save_path, f"audio_{i:02}_*.wav")} -b 16 {os.path.join(audio_save_path, f"concat_{i:02}.wav")}')
    print("Done")
print(f"Saving output to '{os.path.join(audio_save_path, output_filename)}'... ", end='')
os.system(f'sox "{os.path.join(audio_save_path, "concat_*.wav")}" -b 16 "{os.path.join(audio_save_path, output_filename)}"') # merge the merged files into a final output. bit depth of 16 required to go over 4 hour length
print("Done.")<jupyter_output><empty_output><jupyter_text>Show saved Postnet Outputs<jupyter_code>from glob import glob
import random
import numpy as np
import matplotlib
import torch
%matplotlib inline
import matplotlib.pylab as plt
import IPython.display as ipd

filepath = "/media/cookie/Samsung 860 QVO/ClipperDatasetV2/SlicedDialogue"
#filepath = "/media/cookie/Samsung 860 QVO/ClipperDatasetV2"
files = sorted(glob(filepath+"/**/*.mel.npy", recursive=True))
start = int(random.random() * len(files))
file_count = 10
for i in range(start, start+file_count):
    #file = random.choice(files)
    file = files[i]
    H = np.load(file)
    H = (H+5.2)*0.5
    %matplotlib inline
    fig = plt.figure(figsize=(12, 4))

    ax = fig.add_subplot(111)
    ax.set_title(file+"\n"+str(round(np.size(H,1)*(600/48000), 2)))
    plt.imshow(H, cmap='inferno', origin='lower',
                   interpolation='none')
    ax.set_aspect('equal')

    cax = fig.add_axes([0.12, 0.1, 0.78, 0.8])
    cax.get_xaxis().set_visible(False)
    cax.get_yaxis().set_visible(False)
    cax.patch.set_alpha(0)
    cax.set_frame_on(False)
    plt.colorbar(orientation='vertical')
    plt.show()
    
    # again
    file = file.replace(".mel.npy",".npy")
    H = np.load(file)
    H = (H+5.2)*0.5
    %matplotlib inline
    fig = plt.figure(figsize=(12, 4))

    ax = fig.add_subplot(111)
    ax.set_title(file+"\n"+str(round(np.size(H,1)*(600/48000), 2)))
    plt.imshow(H, cmap='inferno', origin='lower',
                   interpolation='none')
    ax.set_aspect('equal')
    
    cax = fig.add_axes([0.12, 0.1, 0.78, 0.8])
    cax.get_xaxis().set_visible(False)
    cax.get_yaxis().set_visible(False)
    cax.patch.set_alpha(0)
    cax.set_frame_on(False)
    plt.colorbar(orientation='vertical')
    plt.show()<jupyter_output><empty_output><jupyter_text>Size of GT Spectrograms<jupyter_code>from glob import glob
import os
import matplotlib
%matplotlib inline
import matplotlib.pylab as plt
import IPython.display as ipd

#filepath = "/media/cookie/Samsung 860 QVO/ClipperDatasetV2/SlicedDialogue/FiM/S1/s1e1"
filepath = "/media/cookie/Samsung 860 QVO/ClipperDatasetV2"
files = sorted(glob(filepath+"/**/*.npy", recursive=True))
total_size = 0
min_duration = 0.6
SR = 48000
BD = 2 # 16 bits
for path in files:
    file_size = os.stat(path).st_size
    if file_size > (SR*BD*min_duration):
        total_size+=file_size

duration = total_size / (SR*BD)
duration_min = duration/60
duration_hrs = duration/3600
total_size_MB = total_size / (1024**3)
print(f"{total_size_MB} GB")<jupyter_output><empty_output><jupyter_text>Size & Duration of Wavs<jupyter_code>from glob import glob
import os
import matplotlib
%matplotlib inline
import matplotlib.pylab as plt
import IPython.display as ipd
def get_stats(filepath, ext='.wav'):
    import soundfile as sf
    files = sorted(glob(filepath+f"/**/*{ext}", recursive=True))
    total_size = 0
    total_duration = 0
    min_duration = 0.6
    SR = 48000
    BD = 2 # 16 bits
    for path in files:
        file_size = os.stat(path).st_size
        audio, samplerate = sf.read(path)
        if len(audio)/samplerate > min_duration:
            total_size+=file_size
            total_duration+=len(audio)/samplerate
    duration_min = total_duration/60
    duration_hrs = total_duration/3600
    print(f"{total_duration} seconds = {duration_min} minutes = {duration_hrs} hours")
    total_size_MB = total_size / (1024**3)
    print(f"{total_size_MB} GB of wavs")

get_stats(r"/media/cookie/Samsung 860 QVO/ClipperDatasetV2/Blizzard2011", ext='.wav')
print(torch.cuda.current_device())
print(torch.cuda.device_count())<jupyter_output><empty_output><jupyter_text>Testing FP16 vs FP32 individual Tensor Operations<jupyter_code>import torch<jupyter_output><empty_output><jupyter_text>tensor.inverse()<jupyter_code>%%timeit
weight = torch.tensor([[1,2,3],
                       [4,5,6],
                       [7,8,9]]).cuda().half()
#########################################################
def b_inv(b_mat):
    eye = b_mat.new_ones(b_mat.size(-1)).diag().expand_as(b_mat).float()
    b_inv, _ = torch.solve(eye.float(), b_mat.float())
    return b_inv
fp16 = b_inv(weight.squeeze())
#########################################################
print(fp16)
%%timeit
weight = torch.tensor([[1,2,3],
                       [4,5,6],
                       [7,8,9]]).cuda().float()
#########################################################
fp32 = weight.squeeze().inverse()
#########################################################
print(fp32)
#assert torch.equal(fp32.float(), fp16.float())<jupyter_output><empty_output><jupyter_text>tensor.slogdet()<jupyter_code>%%timeit
weight = torch.tensor([[1,2,3],
                       [4,5,6],
                       [7,8,9]]).unsqueeze(0).repeat(1000,2,2).cuda().float()
#########################################################
log_det_W = weight.squeeze().slogdet()[1]
%%timeit
weight = torch.tensor([[1,2,3],
                       [4,5,6],
                       [7,8,9]]).unsqueeze(0).repeat(1000,2,2).cuda().float()
#########################################################
log_det_W = weight.squeeze().logdet()
%%timeit
weight = torch.tensor([[1,2,3],
                       [4,5,6],
                       [7,8,9]]).unsqueeze(0).repeat(1000,2,2).cuda().float()
#########################################################
log_det_W = weight.squeeze().det().log()
%%timeit -n 10000
weight = torch.tensor([[1,2,3],
                       [4,5,6],
                       [7,8,9]]).unsqueeze(0).repeat(500,1,1).cuda().half()
#########################################################
log_det_W = weight.squeeze().float().det().log().half()
%%timeit -n 10000
weight = torch.tensor([[1,2,3],
                       [4,5,6],
                       [7,8,9]]).unsqueeze(0).repeat(500,1,1).cuda().half()
#########################################################
log_det_W = weight.squeeze().float().det().half().log()
weight = torch.tensor([[1,2,3],
                       [4,5,6],
                       [7,8,9]]).cuda().half()
#########################################################
log_det_W = weight.squeeze().float().det().half().log()
print(log_det_W)
weight = torch.tensor([[1,2,3],
                       [4,5,6],
                       [7,8,9]]).cuda()
weight.half()
weight.type()<jupyter_output><empty_output><jupyter_text>Evaluate Multiple WaveGlow Models<jupyter_code>import torch
from denoiser import Denoiser
from glob import glob
from random import random
import os
import numpy as np
import soundfile as sf
import matplotlib
import matplotlib.pylab as plt
import IPython.display as ipd
from layers import TacotronSTFT, STFT
from utils import load_wav_to_torch
from hparams import create_hparams
%matplotlib inline
def load_mel(path):
    audio, sampling_rate = load_wav_to_torch(path)
    if sampling_rate != stft.sampling_rate:
        raise ValueError("{} {} SR doesn't match target {} SR".format(
            sampling_rate, stft.sampling_rate))
    audio_norm = audio / hparams.max_wav_value
    audio_norm = audio_norm.unsqueeze(0)
    audio_norm = torch.autograd.Variable(audio_norm, requires_grad=False)
    melspec = stft.mel_spectrogram(audio_norm)
    return melspec
def disp_mel(H, desc=''):
    %matplotlib inline
    fig = plt.figure(figsize=(6, 4))

    ax = fig.add_subplot(111)
    ax.set_title(desc+"\n"+str(round(np.size(H,1)*(600/48000), 2)))
    plt.imshow(H, cmap='inferno', origin='lower',
                   interpolation='none')
    ax.set_aspect('equal')

    cax = fig.add_axes([0.12, 0.1, 0.78, 0.8])
    cax.get_xaxis().set_visible(False)
    cax.get_yaxis().set_visible(False)
    cax.patch.set_alpha(0)
    cax.set_frame_on(False)
    #plt.colorbar(orientation='vertical')
    plt.show()
def plot_audio_spec(audio, sampling_rate=48000):
    if sampling_rate != stft.sampling_rate:
        raise ValueError("{} {} SR doesn't match target {} SR".format(
            sampling_rate, stft.sampling_rate))
    audio_norm = audio / hparams.max_wav_value
    audio_norm = audio_norm.unsqueeze(0)
    audio_norm = torch.autograd.Variable(audio_norm, requires_grad=False)
    melspec = stft.mel_spectrogram(audio_norm)
    return melspec


def load_mel(path):
    audio, sampling_rate = load_wav_to_torch(path)
    if sampling_rate != stft.sampling_rate:
        raise ValueError("{} {} SR doesn't match target {} SR".format(
            sampling_rate, stft.sampling_rate))
    audio_norm = audio / hparams.max_wav_value
    audio_norm = audio_norm.unsqueeze(0)
    audio_norm = torch.autograd.Variable(audio_norm, requires_grad=False)
    melspec = stft.mel_spectrogram(audio_norm)
    return melspec


def load_model(waveglow_path):
    from efficient_model import WaveGlow
    from efficient_util import remove_weight_norms
    import json
    data = r"""{
        "train_config": {
            "fp16_run": false,
            "output_directory": "outdir_EfficientBaseline",
            "epochs": 1000,
            "learning_rate": 1e-4,
            "sigma": 1.0,
            "iters_per_checkpoint": 2000,
            "batch_size": 30,
            "seed": 1234,
            "checkpoint_path": "outdir_EfficientBaseline/waveglow_1425",
            "with_tensorboard": true
        },
        "data_config": {
            "training_files": "map_0_GT.txt",
            "segment_length": 19200,
            "sampling_rate": 48000,
            "filter_length": 2400,
            "hop_length": 600,
            "win_length": 2400,
            "mel_fmin": 0.0,
            "mel_fmax": 16000.0
        },
        "dist_config": {
            "dist_backend": "nccl",
            "dist_url": "tcp://127.0.0.1:54321"
        },
        "waveglow_config": {
            "n_mel_channels": 160,
            "n_flows": 12,
            "n_group": 8,
            "n_early_every": 4,
            "n_early_size": 2,
            "memory_efficient": false,
            "WN_config": {
                "dilation_channels":256,
                "residual_channels":256,
                "skip_channels":256,
                "n_layers": 9,
                "radix": 3,
                "bias": true
            }
        }
    }"""
    config = json.loads(data)
    train_config = config["train_config"]
    global data_config
    data_config = config["data_config"]
    global dist_config
    dist_config = config["dist_config"]
    global waveglow_config
    waveglow_config = { 
        **config["waveglow_config"], 
        'win_length': data_config['win_length'],
        'hop_length': data_config['hop_length']
    }
    waveglow = WaveGlow(**waveglow_config).cuda()
    waveglow_dict = torch.load(waveglow_path)['model'].state_dict()
    waveglow.load_state_dict(waveglow_dict)
    waveglow.apply(remove_weight_norms)
    waveglow.cuda().eval()#.half()
    #for k in waveglow.convinv:
    #    k.float()
    #denoiser = Denoiser(waveglow)
    waveglow_iters = torch.load(waveglow_path)['iteration']
    print(waveglow_iters, "iterations")
    return waveglow, waveglow_iters


def waveglow_infer(mel_outputs_postnet, sigma_, iters=''):
    current_audio = waveglow.infer(mel_outputs_postnet, sigma=sigma_).unsqueeze(0)
    audio.append(current_audio)
    print("sigma = {}".format(sigma_)); ipd.display(ipd.Audio(audio[len(audio)-1][0].data.cpu().numpy(), rate=hparams.sampling_rate))
    maxv = np.iinfo(np.int16).max
    sf.write("infer/temp.wav", (np.swapaxes(audio[len(audio)-1].cpu().numpy(),0,1) * maxv).astype(np.int16), hparams.sampling_rate)
    disp_mel(load_mel("infer/temp.wav").squeeze(), desc=file_path+f"\nAfter WaveGlow {iters}\nSigma {sigma_}")


def waveglow_infer_filepath(file_path, sigma_, iters=''):
    mel_outputs_postnet = np.load(file_path)
    mel_outputs_postnet = (mel_outputs_postnet+5.2)*0.5 # shift values between approx -4 and 4
    mel_outputs_postnet = torch.from_numpy(mel_outputs_postnet).unsqueeze(0).cuda()#.half()
    audio = []
    with torch.no_grad():
        waveglow_infer(mel_outputs_postnet, sigma_, iters=iters)

audio = []
hparams = create_hparams()
stft = TacotronSTFT(hparams.filter_length, hparams.hop_length, hparams.win_length,
                    hparams.n_mel_channels, hparams.sampling_rate, hparams.mel_fmin,
                    hparams.mel_fmax)
filepath = "/media/cookie/Samsung 860 QVO/ClipperDatasetV2/SlicedDialogue"
files = sorted(glob(filepath+"/**/*.npy", recursive=True))
file_path = files[int(random()*len(files))]
print(file_path)
"""outdir_EfficientBaseline/waveglow_5158
outdir_EfficientBaseline/waveglow_5483
outdir_EfficientBaseline/waveglow_5666
outdir_EfficientBaseline/waveglow_7264
outdir_EfficientBias/waveglow_252
outdir_EfficientBias/waveglow_655
outdir_EfficientBias/waveglow_1003
outdir_EfficientBias/waveglow_1935
outdir_EfficientBias/waveglow_11346
outdir_EfficientBias/waveglow_11863
outdir_EfficientBias/waveglow_12118
outdir_EfficientBias/waveglow_12282
outdir_EfficientBias/waveglow_14197
outdir_EfficientBias/waveglow_16058"""
model_paths = r"""
outdir_EfficientBias/waveglow_365
""".split("\n")

disp_mel(load_mel(file_path.replace(".mel.npy",".wav").replace(".npy",".wav")).squeeze(), desc=file_path.replace(".mel.npy",".wav").replace(".npy",".wav")+f"\nGround Truth")
for model_path in [path for path in model_paths if path]:
    waveglow, iters = load_model(f"/media/cookie/Samsung 860 QVO/TTCheckpoints/waveglow/{model_path}")
    waveglow_infer_filepath(file_path, 0.9, iters=iters)<jupyter_output><empty_output><jupyter_text>WaveGlow GTA Inference<jupyter_code>import torch
import torch.nn as nn
from denoiser import Denoiser
from glob import glob
from random import random
import os
import numpy as np
import soundfile as sf
import matplotlib
import matplotlib.pylab as plt
import IPython.display as ipd
from layers import TacotronSTFT, STFT
from utils import load_wav_to_torch
from hparams import create_hparams
from shutil import copyfile

from waveglow_utils import PreEmphasis, InversePreEmphasis
%matplotlib inline
def load_mel(path):
    audio, sampling_rate = load_wav_to_torch(path)
    if sampling_rate != stft.sampling_rate:
        raise ValueError("{} {} SR doesn't match target {} SR".format(
            sampling_rate, stft.sampling_rate))
    audio_norm = audio / hparams.max_wav_value
    audio_norm = audio_norm.unsqueeze(0)
    audio_norm = torch.autograd.Variable(audio_norm, requires_grad=False)
    melspec = stft.mel_spectrogram(audio_norm)
    return melspec
def disp_mel(H, desc=''):
    %matplotlib inline
    fig = plt.figure(figsize=(12, 4))

    ax = fig.add_subplot(111)
    ax.set_title(desc+"\nLength: "+str(round(np.size(H,1)*(600/48000), 2)))
    plt.imshow(H, cmap='inferno', origin='lower',
                   interpolation='none')
    ax.set_aspect('equal')

    cax = fig.add_axes([0.12, 0.1, 0.78, 0.8])
    cax.get_xaxis().set_visible(False)
    cax.get_yaxis().set_visible(False)
    cax.patch.set_alpha(0)
    cax.set_frame_on(False)
    #plt.colorbar(orientation='vertical')
    plt.show()
def plot_audio_spec(audio, sampling_rate=48000):
    if sampling_rate != stft.sampling_rate:
        raise ValueError("{} {} SR doesn't match target {} SR".format(
            sampling_rate, stft.sampling_rate))
    audio_norm = audio / hparams.max_wav_value
    audio_norm = audio_norm.unsqueeze(0)
    audio_norm = torch.autograd.Variable(audio_norm, requires_grad=False)
    melspec = stft.mel_spectrogram(audio_norm)
    return melspec
def load_mel(path):
    audio, sampling_rate = load_wav_to_torch(path)
    if sampling_rate != stft.sampling_rate:
        raise ValueError("{} {} SR doesn't match target {} SR".format(
            sampling_rate, stft.sampling_rate))
    audio_norm = audio / hparams.max_wav_value
    audio_norm = audio_norm.unsqueeze(0)
    audio_norm = torch.autograd.Variable(audio_norm, requires_grad=False)
    melspec = stft.mel_spectrogram(audio_norm)
    return melspec
hparams = create_hparams()
stft = TacotronSTFT(hparams.filter_length, hparams.hop_length, hparams.win_length,
                    hparams.n_mel_channels, hparams.sampling_rate, hparams.mel_fmin,
                    hparams.mel_fmax)
#stft_wideband = TacotronSTFT(hparams.filter_length, hparams.hop_length, 1024,
#                    hparams.n_mel_channels, hparams.sampling_rate, hparams.mel_fmin,
#                    hparams.mel_fmax)<jupyter_output><empty_output><jupyter_text>Nvidia WaveGlow<jupyter_code>def load_waveglow(waveglow_path):
    waveglow_dict = torch.load(waveglow_path)
    waveglow = waveglow_dict['model']
    waveglow_iters = int(waveglow_dict['iteration'])
    waveglow.cuda().eval().half()
    denoiser = Denoiser(waveglow)
    print(waveglow_iters, "iterations")
    if not hasattr(waveglow, "spect_scaling"):
        setattr(waveglow, "spect_scaling", False)
    return waveglow, denoiser, waveglow_iters

def load_waveglow_yoyo(waveglow_path):
    waveglow_dict = torch.load(waveglow_path)
    waveglow = waveglow_dict['model']
    waveglow_iters = int(waveglow_dict['iteration'])
    waveglow.cuda().eval()#.half()
    denoiser = None#Denoiser(waveglow)
    print(waveglow_iters, "iterations")
    return waveglow, denoiser, waveglow_iters
waveglow_dict = torch.load("/media/cookie/Samsung 860 QVO/TTCheckpoints/waveglow/outdir_EfficientLarge/best_model")
waveglow = waveglow_dict['model']
waveglow_iters = int(waveglow_dict['iteration'])
waveglow.cuda().eval()#.half()
denoiser = None#Denoiser(waveglow)
print(waveglow_iters, "iterations")
print(waveglow.WNs)
print(waveglow.WNs[0].WN)<jupyter_output><empty_output><jupyter_text>Select Random File<jupyter_code>def waveglow_infer(mel_outputs_postnet, sigma_, deempathsis, root_dir='infer', clip_folder='', filename=''):
    current_audio = waveglow.infer(mel_outputs_postnet, sigma=sigma_)
    if len(current_audio.shape) == 1:
        current_audio = current_audio.unsqueeze(0)
    if deempathsis:
        deempthasis_filter = InversePreEmphasis(float(deempathsis)) # TODO, replace with something lightweight.
        current_audio = deempthasis_filter(current_audio.cpu().float().unsqueeze(0)).squeeze(0).cuda()
    audio.append(current_audio)
    
    # Show Audio for Listening in Notebook
    #ipd.display(ipd.Audio(audio[len(audio)-1][0].data.cpu().numpy(), rate=hparams.sampling_rate))
    
    # Save Audio
    local_fpath = os.path.join(root_dir, clip_folder, filename) # local filepath
    local_dpath = os.path.join(root_dir, clip_folder) # local directory path
    os.makedirs(local_dpath, exist_ok=True) # ensure local directory exists
    maxv = np.iinfo(np.int16).max # get max int16 value
    sf.write(os.path.join(root_dir, "temp.wav"), (np.swapaxes(audio[len(audio)-1].cpu().numpy(),0,1) * maxv).astype(np.int16), hparams.sampling_rate) # write audio to temp
    
    # Get MSE and MAE
    waveglow_spect = load_mel(os.path.join(root_dir, "temp.wav")).squeeze() # load spectrogram from wav file.
    waveglow_spect_lossy = (waveglow_spect.unsqueeze(0).cuda().half()[:,:,:mel_outputs_postnet.shape[-1]])#+5.2)*0.5 # move spectrogram to GPU, reshape and normalize within -4, 4.
    MSE = (nn.MSELoss()(waveglow_spect_lossy, mel_outputs_postnet)).item() # get MSE (Mean Squared Error) between Ground Truth and WaveGlow inferred spectrograms.
    MAE = (nn.L1Loss()(waveglow_spect_lossy, mel_outputs_postnet)).item() # get MAE (Mean Absolute Error) between Ground Truth and WaveGlow inferred spectrograms.
    
    #sf.write(local_fpath+f"-MSE_{round(MSE,4)}.wav", (np.swapaxes(audio[len(audio)-1].cpu().numpy(),0,1) * maxv).astype(np.int16), hparams.sampling_rate) # write audio to fpath
    sf.write(local_fpath+f"-MSE_{round(MSE,4)}.wav", np.swapaxes(audio[len(audio)-1].cpu().numpy(),0,1), hparams.sampling_rate, "PCM_16") # write audio to fpath
    # Show Spect
    #disp_mel(waveglow_spect, desc=f"\nAfter WaveGlow\nSigma: {sigma_}\nMSELoss: {MSE}") # Show Plot in Notebook
    return MSE, MAE
filepath = "/media/cookie/Samsung 860 QVO/ClipperDatasetV2/SlicedDialogue"
files = sorted(glob(filepath+"/**/*__*.npy", recursive=True))
waveglow_infer_paths = [
    "/media/cookie/Samsung 860 QVO/ClipperDatasetV2/SlicedDialogue/Special source/s6e24/00_04_52_Rainbow_Neutral__That's a mighty big claim considering everypony here is an amazingly awesome crazy good flyer.mel.npy"
]
for i in range(19):
    file_path = files[int(random()*len(files))]
    waveglow_infer_paths.append(file_path)
print("".join([x+"\n" for x in waveglow_infer_paths]))# Print each entry in multiple lines.
x="""
# Baseline, 12 Flow, 256 Channel, 8 Layer, 0.00 Empthasis
outdir_twilight9/waveglow_140900|1|0|0.00|0
outdir_twilight9/waveglow_152256|1|0|0.00|0
outdir_twilight9/waveglow_187539|1|0|0.00|0
outdir_twilight9/waveglow_229258|1|0|0.00|0
outdir_twilight9/waveglow_268276|1|0|0.00|0
outdir_twilight9/waveglow_311477|1|0|0.00|0
outdir_twilight9/waveglow_334361|1|0|0.00|0
outdir_twilight9/waveglow_351862|1|0|0.00|0
outdir_twilight9/waveglow_387823|1|0|0.00|0
outdir_twilight9/waveglow_422080|1|0|0.00|0
# Mini (ReZero), 12 Flow, 128 Channel, 10 Layer, 0.97 Empthasis
outdir_12Flow_128Channel_10Layer/waveglow_10000|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_30000|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_38265|1|0|0.97|0
# Mini, 12 Flow, 128 Channel, 10 Layer, 0.97 Empthasis
outdir_12Flow_128Channel_10Layer/waveglow_2598|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_3979|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_6496|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_13894|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_15659|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_35439|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_40000|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_55133|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_100000|1|0|0.9|07
outdir_12Flow_128Channel_10Layer/waveglow_120000|1|0|0.9|07
# Mini, 16 Flow, 128 Channel, 10 Layer, 0.97 Empthasis
outdir_16Flow_128Channel_10Layer/waveglow_11306|1|0|0.97|0
outdir_16Flow_128Channel_10Layer/waveglow_55528|1|0|0.97|0
outdir_16Flow_128Channel_10Layer/waveglow_57259|1|0|0.97|0
outdir_16Flow_128Channel_10Layer/waveglow_69333|1|0|0.97|0
outdir_16Flow_128Channel_10Layer/waveglow_77184|1|0|0.97|0
# Mini, 24 Flow, 128 Channel, 10 Layer, 0.97 Empthasis
outdir_24Flow_128Channel_10Layer/waveglow_10000|1|0|0.97|0
outdir_24Flow_128Channel_10Layer/waveglow_66559|1|0|0.97|0
outdir_24Flow_128Channel_10Layer/waveglow_70000|1|0|0.97|0
outdir_24Flow_128Channel_10Layer/waveglow_80000|1|0|0.97|0
outdir_24Flow_128Channel_10Layer/waveglow_96555|1|0|0.97|0
outdir_24Flow_128Channel_10Layer/waveglow_108044|1|0|0.97|0
# LARGE, 16 Flow, 512 Channel, 10 Layer, 0.97 Empthasis
outdir_EfficientLarge/waveglow_60000|1|0|0.97|1
outdir_EfficientLarge/waveglow_70000|1|0|0.97|1
outdir_EfficientLarge/waveglow_80000|1|0|0.97|1
outdir_EfficientLarge/waveglow_90000|1|0|0.97|1
outdir_EfficientLarge/waveglow_120119|1|0|0.97|1
outdir_EfficientLarge/best_model|1|0|0.97|1
outdir_EfficientLarge/waveglow_127217|1|0|0.97|1
# Baseline (Nancy Datset Only), 12 Flow, 256 Channel, 8 Layer, 0.00 Empthasis
"""

# path|normalize(-4 to 4)|mu_law_quantization|de-empthasis|yoyololicon version
waveglow_paths = """
outdir_twilight9/waveglow_140900|1|0|0.00|0
outdir_twilight9/waveglow_152256|1|0|0.00|0
outdir_twilight9/waveglow_187539|1|0|0.00|0
outdir_twilight9/waveglow_229258|1|0|0.00|0
outdir_twilight9/waveglow_268276|1|0|0.00|0
outdir_twilight9/waveglow_311477|1|0|0.00|0
outdir_twilight9/waveglow_334361|1|0|0.00|0
outdir_twilight9/waveglow_351862|1|0|0.00|0
outdir_twilight9/waveglow_387823|1|0|0.00|0
outdir_twilight9/waveglow_422080|1|0|0.00|0
outdir_12Flow_128Channel_10Layer/waveglow_10000|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_30000|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_38265|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_2598|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_3979|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_6496|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_13894|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_15659|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_35439|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_40000|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_55133|1|0|0.97|0
outdir_12Flow_128Channel_10Layer/waveglow_100000|1|0|0.9|07
outdir_12Flow_128Channel_10Layer/waveglow_120000|1|0|0.9|07
outdir_16Flow_128Channel_10Layer/waveglow_11306|1|0|0.97|0
outdir_16Flow_128Channel_10Layer/waveglow_55528|1|0|0.97|0
outdir_16Flow_128Channel_10Layer/waveglow_57259|1|0|0.97|0
outdir_16Flow_128Channel_10Layer/waveglow_69333|1|0|0.97|0
outdir_16Flow_128Channel_10Layer/waveglow_77184|1|0|0.97|0
outdir_24Flow_128Channel_10Layer/waveglow_10000|1|0|0.97|0
outdir_24Flow_128Channel_10Layer/waveglow_66559|1|0|0.97|0
outdir_24Flow_128Channel_10Layer/waveglow_70000|1|0|0.97|0
outdir_24Flow_128Channel_10Layer/waveglow_80000|1|0|0.97|0
outdir_24Flow_128Channel_10Layer/waveglow_96555|1|0|0.97|0
outdir_24Flow_128Channel_10Layer/waveglow_108044|1|0|0.97|0
outdir_EfficientLarge/waveglow_60000|1|0|0.97|1
outdir_EfficientLarge/waveglow_70000|1|0|0.97|1
outdir_EfficientLarge/waveglow_80000|1|0|0.97|1
outdir_EfficientLarge/waveglow_90000|1|0|0.97|1
outdir_EfficientLarge/waveglow_120119|1|0|0.97|1
outdir_EfficientLarge/best_model|1|0|0.97|1
outdir_EfficientLarge/waveglow_127217|1|0|0.97|1
outdir_NancyOnly/best_model|0|0|0.00|0
"""[1:-1].split("\n")

print("Missing Checkpoints:")
print("\n".join([x.split("|")[0] for x in waveglow_paths if not os.path.exists("../tacotron2/waveglow_latest/"+x.split("|")[0])]))
waveglow_paths = [x for x in waveglow_paths if os.path.exists("../tacotron2/waveglow_latest/"+x.split("|")[0])]

with torch.no_grad():
    for waveglow_meta in waveglow_paths:
        print("--------------------------------------------------------")
        waveglow_path, normalize_spec, mu_law_quantization, deempthasis_strength, b_yoyololicon_model = waveglow_meta.split("|")
        waveglow_info = str("_".join(waveglow_path.split("/")[0].split("_")[1:]))
        print(waveglow_meta)
        if b_yoyololicon_model:
            waveglow, denoiser, waveglow_current_iter = load_waveglow_yoyo("../tacotron2/waveglow_latest/"+waveglow_path)
        else:
            waveglow, denoiser, waveglow_current_iter = load_waveglow("../tacotron2/waveglow_latest/"+waveglow_path)
        
        if not hasattr(waveglow, "spect_scaling"):
            setattr(waveglow, "spect_scaling", False)
        
        audio = []
        total_MAE = total_MSE = 0
        best_MAE = best_MSE = 9e9
        worst_MAE = worst_MSE = -9e9
        for file_path in waveglow_infer_paths:
            #print(f"FILE: {file_path}") # Print the file path
            basename = os.path.splitext(os.path.basename(file_path.replace(".mel.npy",".npy")))[0] # filename without ext
            
            mel_outputs_postnet = np.load(file_path) # Load Tacotron2 Postnet Outputs
            if int(normalize_spec):
                mel_outputs_postnet = (mel_outputs_postnet+5.2)*0.5 # shift values between approx -4 and 4
            
            #disp_mel(load_mel(file_path.replace(".mel.npy",".wav").replace(".npy",".wav")).squeeze(), desc=f"\nGround Truth") # Display Ground Truth Spectrogram
            #disp_mel(mel_outputs_postnet, desc="\nThis is the original Postnet output from Tacotron") # Display Tacotron GTA Spectrogram
            
            mel_outputs_postnet = np.load(file_path.replace(".mel.npy",".npy")) # Load Ground Truth Spectrogram for inference by WaveGlow.
            if int(normalize_spec):
                mel_outputs_postnet = (mel_outputs_postnet+5.2)*0.5 # shift values between approx -4 and 4, speeds up initial training
            mel_outputs_postnet = torch.from_numpy(mel_outputs_postnet).unsqueeze(0).cuda() # prep tensor for WaveGlow.
            if not b_yoyololicon_model:
                mel_outputs_postnet = mel_outputs_postnet.half()
            
            sigma = 0.9
            MSE, MAE = waveglow_infer(mel_outputs_postnet, sigma, deempthasis_strength, clip_folder=f'{basename}', filename=f'iter_{waveglow_current_iter:07}-sigma_{sigma}-empth_{float(deempthasis_strength)}')
            sigma = 0.95
            MSE, MAE = waveglow_infer(mel_outputs_postnet, sigma, deempthasis_strength, clip_folder=f'{basename}', filename=f'iter_{waveglow_current_iter:07}-sigma_{sigma}-empth_{float(deempthasis_strength)}')
            sigma = 1.0
            MSE, MAE = waveglow_infer(mel_outputs_postnet, sigma, deempthasis_strength, clip_folder=f'{basename}', filename=f'iter_{waveglow_current_iter:07}-sigma_{sigma}-empth_{float(deempthasis_strength)}')
            
            if not os.path.exists(os.path.join("infer", basename, f'GroundTruth.wav')):
                copyfile(os.path.splitext(file_path.replace(".mel.npy",".npy"))[0]+".wav", os.path.join("infer", basename, f'GroundTruth.wav'))
            total_MSE+=MSE
            best_MSE = min(best_MSE, MSE)
            worst_MSE = max(worst_MSE, MSE)
            total_MAE+=MAE
            best_MAE = min(best_MAE, MAE)
            worst_MAE = max(worst_MAE, MAE)
        print(f"Average MSE: {total_MSE/len(waveglow_infer_paths)} Best MSE: {best_MSE} Worst MSE: {worst_MSE}")
        print(f"Average MAE: {total_MAE/len(waveglow_infer_paths)} Best MAE: {best_MAE} Worst MAE: {worst_MAE}")
%%timeit
x = torch.rand(4096,1024,2,1, device="cuda:0")
x = x + 10
x = x.squeeze()
from time import sleep
sleep(10)
%%timeit
x = torch.rand(4096,1024,2,1, device="cuda:0")
x = x + 10
setattr(waveglow, "spect_scaling", False)
list(waveglow.parameters())[0].type()
import torch
batch_dim = 2
n_group = 8
audio = torch.arange(32).repeat(batch_dim, 1)
print(audio)
audio = audio.view(batch_dim, -1, n_group).transpose(1, 2)
print("shape =", audio.shape)
print(audio)
x = torch.rand(2,6,1)
print(x)
y_0, y_1 = x.chunk(2,1)
print("y_0 =\n", y_0)
print("y_1 =\n", y_1)

n_half = int(x.size(1)/2)
y_0 = x[:,:n_half,:]
y_1 = x[:,n_half:,:]
print("y_0 =\n", y_0)
print("y_1 =\n", y_1)
print(x.data)
x = torch.ones(5)
print(x)
x.mul_(2).add_(-5)
print(x.view(-1))
x = x.unsqueeze(0)
print(x.view(-1))
x = (torch.rand(5)-0.5)*2
print(x)
x.max().item()
y = x*x
print(y)
y = torch.abs(x)
print(y)
torch.cuda.empty_cache()
print(
    torch.cuda.get_device_properties("cuda:0").total_memory/1024**3,"\n",
    torch.cuda.memory_allocated("cuda:0")/1024**3,"\n",
    torch.cuda.max_memory_allocated("cuda:0")/1024**3,"\n",
    torch.cuda.memory_reserved("cuda:0")/1024**3,"\n",
    torch.cuda.max_memory_reserved("cuda:0")/1024**3,"\n",
    torch.cuda.memory_summary("cuda:0")
)
denoise_strength = 10
for i in audio:
    audio_denoised = denoiser(i, strength=denoise_strength)[:, 0]
    ipd.display(ipd.Audio(audio_denoised.cpu().numpy(), rate=hparams.sampling_rate))
maxv = np.iinfo(np.int16).max
sf.write("infer/temp.wav", (np.swapaxes(audio_denoised.cpu().numpy(),0,1) * maxv).astype(np.int16), hparams.sampling_rate)
disp_mel(load_mel("infer/temp.wav").squeeze(), desc=file_path+f"\nAfter Denoise\nDenoise Strength {denoise_strength}")
x = torch.rand(5,5)
print(x)
y = x*torch.tensor([0,1,0,1,0])
print(y)
# masked_fill_ should be a little more performant.
batch_size = 6
truncated_length = 100
lengths = torch.tensor([268, 239, 296, 148, 87, 453, 601, 602, 603, 604, 605, 606, 607, 608, 609])
processed = 0

batch_lengths = lengths[:batch_size]

for i in range(4):
    print("-"*100)
    print(batch_lengths)
    print((batch_lengths-truncated_length))
    print((batch_lengths-truncated_length)[batch_lengths-truncated_length>0])
    print((batch_lengths-truncated_length)[batch_lengths-truncated_length>0].shape[0])
    print(batch_size - (batch_lengths-truncated_length)[batch_lengths-truncated_length>0].shape[0])
    
    #batch_lengths = (batch_lengths-truncated_length)[batch_lengths-truncated_length>0]
    print(batch_lengths)
    print("processed =",processed)
    processed+=batch_size-((batch_lengths-truncated_length)[batch_lengths-truncated_length>0]).shape[0]
    batch_lengths = torch.cat((batch_lengths, lengths[processed+batch_size:processed+batch_size+(batch_size-batch_lengths.shape[0])]), 0)
    print(batch_lengths)
batch_size = 6
processed = 0
lengths = torch.tensor([0,1,2,3,4,5,6,7,8,9])
x = lengths[:batch_size]
print(x)
print(x[x<4].shape[0])
x[x<4] = lengths[processed+batch_size:processed+batch_size+x[x<4].shape[0]]
print(x)<jupyter_output><empty_output><jupyter_text>Check Pre-empthasis for Audio Quality<jupyter_code>import torch
import torch.nn as nn
from glob import glob
from random import random
import soundfile as sf
import numpy as np
import IPython.display as ipd
from waveglow_utils import PreEmphasis, InversePreEmphasis
preempthasis_strength = 0.97
filepath = "/media/cookie/Samsung 860 QVO/ClipperDatasetV2/SlicedDialogue"
files = sorted(glob(filepath+"/**/*__*.wav", recursive=True))

filepath = "/media/cookie/Samsung 860 QVO/ClipperDatasetV2/VCTK-Corpus-0.92/wav"
files = sorted(glob(filepath+"/**/*.wav", recursive=True))

preempth_filter = PreEmphasis(preempthasis_strength).float()
deempth_filter = InversePreEmphasis(preempthasis_strength).float()
file_path = files[int(random()*len(files))]
print(file_path)
file_path = files[int(random()*len(files))]
print(file_path)

with torch.no_grad():
    audio, sample_rate = sf.read(file_path)
    print(audio.max())
    print(audio.min())
    print("Original")
    ipd.display(ipd.Audio(audio, rate=sample_rate))
    
    sf.write("infer/temp_original.wav", audio, sample_rate)
    disp_mel(load_mel("infer/temp_original.wav").squeeze(), desc=file_path)
    
    import scipy
    from scipy import signal
    sos = signal.butter(10, 60, 'hp', fs=48000, output='sos')
    filtered_audio = signal.sosfilt(sos, audio)
    
    sf.write("infer/temp.wav", filtered_audio, sample_rate)
    disp_mel(load_mel("infer/temp.wav").squeeze(), desc=file_path)
    
    sos = signal.butter(2, 60, 'hp', fs=48000, output='sos')
    filtered_audio = signal.sosfilt(sos, audio)
    
    sf.write("infer/temp.wav", filtered_audio, sample_rate)
    disp_mel(load_mel("infer/temp.wav").squeeze(), desc=file_path)
    
    maxv = np.iinfo(np.int16).max
    audio = deempth_filter((torch.tensor(audio)/maxv).unsqueeze(0).unsqueeze(0).float())
    print(audio.mean())
    print(audio.std())
    print("De-Empthasis")
    ipd.display(ipd.Audio(audio.squeeze()*maxv, rate=sample_rate))
    
    audio, sample_rate = sf.read(file_path)
    audio = preempth_filter((torch.tensor(audio)/maxv).unsqueeze(0).unsqueeze(0).float())
    print("Pre-Empthasis")
    ipd.display(ipd.Audio(audio.squeeze()*maxv, rate=sample_rate))
    
    print("Pre-Empthasis + De-Empthasis")
    audio = deempth_filter(audio)
    ipd.display(ipd.Audio(audio.squeeze()*maxv, rate=sample_rate))
import numpy as np

def mcd(C, C_hat):
    """C and C_hat are NumPy arrays of shape (T, D),
    representing mel-cepstral coefficients.

    """
    K = 10 / np.log(10) * np.sqrt(2)
    return K * np.mean(np.sqrt(np.sum((C - C_hat) ** 2, axis=1)))
mcd(np.array([[0,0.8,0]]),np.array([[0,1,0]]))<jupyter_output><empty_output><jupyter_text>Force Loading Dictionaries<jupyter_code>pretrained = {
    'a':1,
    'b':2,
    'c':3,
    'e':5,
}
modeldict = {
    'c':3,
    'e':4,
    'f':5,
}
dummy_modeldict = {k: v for k,v in pretrained.items() if k in modeldict and pretrained[k] == modeldict[k]}
model_dict_missing = {k: v for k,v in pretrained.items() if k not in modeldict}
model_dict_mismatching = {k: v for k,v in pretrained.items() if k in modeldict and pretrained[k] != modeldict[k]}
pretrained_missing = {k: v for k,v in modeldict.items() if k not in pretrained}
print(list(model_dict_missing.keys()),'does not exist in the current model')
print(list(model_dict_mismatching.keys()),"is the wrong shape and has been reset")
print(list(pretrained_missing.keys()),"doesn't have pretrained weights and is reset")
print(dummy_modeldict)<jupyter_output><empty_output><jupyter_text>Adding Blur to Spectrograms during training<jupyter_code>def load_mel(path):
    audio, sampling_rate = load_wav_to_torch(path)
    if sampling_rate != stft.sampling_rate:
        raise ValueError("{} {} SR doesn't match target {} SR".format(
            sampling_rate, stft.sampling_rate))
    audio_norm = audio / hparams.max_wav_value
    audio_norm = audio_norm.unsqueeze(0)
    audio_norm = torch.autograd.Variable(audio_norm, requires_grad=False)
    melspec = stft.mel_spectrogram(audio_norm)
    return melspec
def disp_mel(H, desc=''):
    %matplotlib inline
    fig = plt.figure(figsize=(12, 6))

    ax = fig.add_subplot(111)
    ax.set_title(desc+"\n"+str(round(np.size(H,1)*(600/48000), 2)))
    plt.imshow(H, cmap='inferno', origin='lower',
                   interpolation='none')
    ax.set_aspect('equal')

    cax = fig.add_axes([0.12, 0.1, 0.78, 0.8])
    cax.get_xaxis().set_visible(False)
    cax.get_yaxis().set_visible(False)
    cax.patch.set_alpha(0)
    cax.set_frame_on(False)
    #plt.colorbar(orientation='vertical')
    plt.show()
import kornia
x = load_mel("/media/cookie/Samsung 860 QVO/ClipperDatasetV2/SlicedDialogue/Special source/s6e24/00_03_22_Rainbow_Neutral__tell ya what. I'll leave the teaching stuff to you And I'll just make sure they stay awake.wav")
scale_embed = torch.rand([1,160])*0.5 + 0.55
x = x*scale_embed.unsqueeze(2)
print(x.shape)
print(scale_embed.shape)
print(scale_embed)
x = x.cuda()
disp_mel(x.squeeze().cpu())

x_strength = 100.0
y_strength = 0.001
filter_cycles = 3
filter = kornia.filters.GaussianBlur2d((3,3),(x_strength,y_strength))

#print(x.shape)
# filter input needs (B,C,H,W)
out = filter(x.unsqueeze(0))
for i in range(filter_cycles-1): out = filter(out)
disp_mel(out.squeeze().cpu())<jupyter_output><empty_output><jupyter_text>Mask for Tacotron Loss<jupyter_code>import torch
def get_mask_from_lengths(lengths):
    max_len = torch.max(lengths).item()
    ids = torch.arange(0, max_len, out=torch.LongTensor(max_len))
    mask = (ids < lengths.unsqueeze(1))
    return mask
x = torch.rand(4,8)
output_lengths = torch.tensor([8,5,2,1])
print(x)
print(output_lengths)
y = ~get_mask_from_lengths(output_lengths)
print(y)
list(x.shape)
y = y.expand_as(x)
print(y)
x.masked_fill_(y, 1e3)
print(x)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13795838574507463, 'fraction_numerical': 0.04550006810188041, 'mean_word_length': 3.9448912483657543, 'pattern_counts': {'":': 40, '<': 161, '<?xml version=': 0, '>': 211, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 8, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 4, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '37', 'dir': 'jupyter-structured-clean-dedup', 'id': '623752', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 48494, 'n_tokens_neox': 44507, 'n_words': 9124, 'path': 'CookieTTS/scripts/inference.ipynb', 'repo_name': 'AstraliteHeart/cookietts', 'size': '124850'}