<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Load libraries2&nbsp;&nbsp;Prepare the dataset  Load libraries<jupyter_code>import librosa
import librosa.display
import IPython.display as ipd
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd<jupyter_output><empty_output><jupyter_text>Prepare the dataset<jupyter_code>data_dir = "./data/"
audio_pairs_df = pd.read_csv(data_dir + "audio_sentence_pairs_test.csv")
audio_pairs_df
import _pickle as cPickle
import preprocessing

TEST_SET = ["val", "test"]
VAL_OR_TEST = 1
MFCC_LEN_LIM = 700

waveform_dir = data_dir + "waveform/"

def create_mfcc_data(student_id, article_id, sentence_id):
    audio_sentence = np.load(waveform_dir + student_id + "/{:02d}/".format(article_id + 1) + str(sentence_id) + ".npy")
    return preprocessing.mfcc(audio_sentence)[:MFCC_LEN_LIM]

data_mfcc = {
    "sentence_I": [],
    "sentence_II": [],
    "same_speaker": []
}

audio_pairs = audio_pairs_df.to_numpy()[VAL_OR_TEST * len(audio_pairs_df) // 2:
                                        (VAL_OR_TEST+1) * len(audio_pairs_df) // 2]
print('Progress: ', end='')
for i in range(len(audio_pairs)):
    row = audio_pairs[i]
    data_mfcc['sentence_I'].append(create_mfcc_data(row[0], row[1], row[2]))
    data_mfcc['sentence_II'].append(create_mfcc_data(row[3], row[4], row[5]))
    data_mfcc['same_speaker'].append(row[0] == row[3])
    if i * 100 // len(audio_pairs) > (i - 1) * 100 // len(audio_pairs):
        print('|', end='')
    
cPickle.dump(data_mfcc, open(data_dir + f"dataset/data_mfcc_{TEST_SET[VAL_OR_TEST]}.pkl", 'wb'), protocol=2)<jupyter_output>Progress: ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
{'fraction_non_alphanumeric': 0.19607843137254902, 'fraction_numerical': 0.016147635524798153, 'mean_word_length': 5.402214022140221, 'pattern_counts': {'":': 3, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '536714', 'license': 'MIT', 'n_tokens_mistral': 703, 'n_tokens_neox': 673, 'n_words': 105, 'path': 'preprocessing_test.ipynb', 'repo_name': 'quocanuetcs/voice-recognition', 'size': '1734'}