<jupyter_start><jupyter_text>Alignment<jupyter_code>inflect_engine = inflect.engine()
model = load_model(model_path)
text = clean_text(text, inflect_engine)
sequence = text_to_sequence(text)
_, mel_outputs_postnet, _, alignments = model.inference(sequence)
data = alignments.float().data.cpu().numpy()[0].T
y_length, x_length = data.shape
plt.imshow(data, aspect='auto', origin='lower', interpolation='none')
print(data.shape)
points = []

for i in range(0,x_length):
    current = list(data[:,i])
    points.append(current.index(max(current)))

plt.scatter(range(0,x_length), points)
plt.plot((0,x_length),(0,y_length))<jupyter_output><empty_output><jupyter_text>Cut trailing points<jupyter_code>def get_highest_confidence_points(data):
    points = []

    for i in range(0,data.shape[1]):
        current = list(data[:,i])
        points.append(current.index(max(current)))
    
    return points
def cut_trailing_points(data):
    highest_point = 0
    highest_point_index = 0
    for i in range(0,data.shape[1]):
        current = list(data[:,i])
        point = current.index(max(current))
        if point > highest_point:
            highest_point = point
            highest_point_index = i
            
    return data[:,:highest_point_index]
data = cut_trailing_points(data)
y_length, x_length = data.shape
points = get_highest_confidence_points(data)

plt.scatter(range(0,x_length), points)
plt.plot((0,x_length),(0,y_length))<jupyter_output><empty_output><jupyter_text>Alignment Scoring<jupyter_code>def plot_scores(scores, points, x_length):
    colours = []
    for i in range(0, x_length):
        region_score = scores[i]
        red = 1-region_score
        colours.append(red)

    plt.scatter(range(0,x_length), points, c=[(red,0,1-red) for red in colours])
def calculate_alignment_scores(data, sample_padding=2):
    y_length, x_length = data.shape
    grad = y_length/x_length
    scores = []

    for x in range(0,x_length):
        y = int(x * grad)
        min_index = max(0, y-sample_padding)
        max_index = min(y_length, y+sample_padding)

        score = sum([data[i,x] for i in range(min_index, max_index)])
        scores.append(score)

    return scores
scores = calculate_alignment_scores(data)
print("Overall alignment score:", sum(scores)/len(scores))
print("Best:", max(scores), "Worst", min(scores))
plot_scores(scores, points, x_length)<jupyter_output>Overall alignment score: 0.3876011150526583
Best: 0.9937464000540785 Worst 0.00020468094953685068<jupyter_text>Confidence scoring<jupyter_code>def calculate_confidence_scores(data):
    y_length, x_length = data.shape
    return [max(data[:,i]) for i in range(x_length)]
scores = calculate_confidence_scores(data)
print("Overall confidence score:", sum(scores)/len(scores))
print("Best:", max(scores), "Worst", min(scores))
plot_scores(scores, points, x_length)<jupyter_output>Overall confidence score: 0.4563400856994685
Best: 0.87784207 Worst 0.1754935<jupyter_text>Gap scoring<jupyter_code>def calculate_gap_scores(data):
    distances = []
    points = get_highest_confidence_points(data) 
    for i in range(0, len(points)-1):
        distances.append(abs(points[i] - points[i+1]))
        
    distances.append(0)
    
    max_distance = max(distances)
    return [1-(dist/max_distance) for dist in distances]
scores = calculate_gap_scores(data)
plot_scores(scores, points, x_length)<jupyter_output><empty_output><jupyter_text>Score checkpoints<jupyter_code>checkpoints_directory = "../../tacotron2/outdir"
num_checkpoints = len([f for f in os.listdir(checkpoints_directory) if os.path.isfile(os.path.join(checkpoints_directory,f))])
print(num_checkpoints)
alignment_scores = []
confidence_scores = []
gap_scores = []

for checkpoint in tqdm(range(1000, num_checkpoints*1000, 1000)):
    model = load_model(os.path.join(checkpoints_directory, f"checkpoint_{checkpoint}"))
    _, _, _, alignments = model.inference(sequence)
    data = alignments.float().data.cpu().numpy()[0].T
    cut_data = cut_trailing_points(data)
    if cut_data.shape[1] > 0:
        data = cut_data
    
    alignment = calculate_alignment_scores(data)
    confidence = calculate_confidence_scores(data)
    gap = calculate_gap_scores(data)
    alignment_scores.append(sum(alignment)/len(alignment))
    confidence_scores.append(sum(confidence)/len(confidence))
    gap_scores.append(sum(gap)/len(gap))
plt.scatter(range(len(alignment_scores)), alignment_scores, color="red")
plt.scatter(range(len(alignment_scores)), confidence_scores, color="blue")
plt.scatter(range(len(gap_scores)), gap_scores, color="green")
final = []
for i in range(len(gap_scores)):
    print(gap_scores[i], confidence_scores[i])
    final.append(gap_scores[i]*confidence_scores[i])
plt.scatter(range(len(gap_scores)), final, color="green")<jupyter_output><empty_output><jupyter_text>Replacing words<jupyter_code>waveglow = load_waveglow("waveglow_256channels_universal_v5.pt")
text = clean_text(text, inflect_engine)
sequence = text_to_sequence(text)
_, mel_outputs_postnet, _, alignments = model.inference(sequence)
data = alignments.float().data.cpu().numpy()[0].T
plt.imshow(data, aspect='auto', origin='lower', interpolation='none')
scores = calculate_confidence_scores(data)
sum(scores)/len(scores)
import nltk
nltk.download('wordnet')
from nltk.corpus import wordnet
def get_synonyms(word):
    synonyms = set()
    for syn in wordnet.synsets(word):
        for l in syn.lemmas():
            if "_" not in l.name():
                synonyms.add(l.name().lower())
    synonyms.discard(word)
    return synonyms
def produce_audio(text): 
    sequence = text_to_sequence(text)
    _, mel_outputs_postnet, _, _ = model.inference(sequence)

    with torch.no_grad():
        audio = waveglow.infer(mel_outputs_postnet, sigma=0.666)
        audio = audio[0].data.cpu().numpy()

    return audio
text = "There is a way to measure the acute emotional intelligence that has never gone out of style."
text = clean_text(text, inflect_engine)
print(text)
audio = produce_audio(text)
ipd.Audio(audio, rate=22050)
import scipy.io.wavfile
scipy.io.wavfile.write('temp.wav',22050,audio)
results = transcribe("temp.wav")
def evalulate_audio(audio):
    scipy.io.wavfile.write('temp.wav',22050,audio)
    results = transcribe("temp.wav")
    original_words = text.split(" ")
    produced_words = results.split(" ")
    return set(original_words) - set(produced_words)
scipy.io.wavfile.write('temp.wav',22050,audio)
results = transcribe("temp.wav")
print(text)
print(results)
def find_replacement_word(text, word):
    synonyms = get_synonyms(word)
    print(word, synonyms)
    if synonyms: 
        for synonym in synonyms:
            new_text = text.replace(word, synonym)
            new_audio = produce_audio(new_text)
            missed_words = evalulate_audio(new_audio)
            if synonym not in missed_words:
                return new_text
    else:
        return text
words_not_produced_correctly = evalulate_audio(audio)
print(words_not_produced_correctly)
for word in words_not_produced_correctly:
    new_text = find_replacement_word(text, word)
audio = produce_audio(new_text)
ipd.Audio(audio, rate=22050)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12649430080622742, 'fraction_numerical': 0.02321378926883514, 'mean_word_length': 4.96106048053024, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '601912', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2632, 'n_tokens_neox': 2417, 'n_words': 426, 'path': 'research/Score Inference.ipynb', 'repo_name': 'Sirtwitter/a', 'size': '7194'}