<jupyter_start><jupyter_text>Imports<jupyter_code>import os

import librosa
import matplotlib.pyplot as plt
import numpy as np
import json
import pandas as pd
from scipy.io import wavfile
import scipy.stats as stats
import re
import asyncio
import time
import nest_asyncio
import parselmouth<jupyter_output><empty_output><jupyter_text>Extract Frequency Features  Get Metadata<jupyter_code>project_root =  os.path.dirname(os.path.dirname(os.getcwd()))
source = os.path.join(project_root, "MLS", "Full_split")

metadata = pd.read_csv(os.path.join(source, "metainfo.csv"))
data = metadata[['SPEAKER','GENDER']]
data_dict = dict(zip(data.SPEAKER, data.GENDER))
data_dict<jupyter_output><empty_output><jupyter_text>Extract Features<jupyter_code>async def get_frequencies(file):
    
    file_path = os.path.join(source, file)
    audio_data = parselmouth.Sound(file_path)
    audio_data = audio_data.values[0]
    sample_rate = 22050

    splited_file = file.split('_')
    if data_dict[int(splited_file[0])] == 'F':
        gender = 0
    if data_dict[int(splited_file[0])] == 'M': 
        gender = 1

    step = int(sample_rate/5) #3200 sampling points every 1/5 sec
    window_frequencies = []

    for i in range(0,len(audio_data),step):
        ft = np.fft.fft(audio_data[i:i+step]) #fft returns the list N complex numbers
        freqs = librosa.fft_frequencies(sr=sample_rate, n_fft=len(ft))
        freqs = np.fft.fftfreq(len(ft)) #fftq tells you the frequencies associated with the coefficients
        imax = np.argmax(np.abs(ft))
        freq = freqs[imax]
        freq_in_hz = abs(freq *sample_rate)
        window_frequencies.append(freq_in_hz)

    return window_frequencies, gender, file
async def get_features(count, file):
    async with sem:
        frequencies, gender, file_name = await get_frequencies(file)

        nobs, minmax, mean, variance, skew, kurtosis =  stats.describe(frequencies)
        median   = np.median(frequencies)
        mode     = stats.mode(frequencies).mode[0]
        std      = np.std(frequencies)
        low,peak = minmax
        q75,q25  = np.percentile(frequencies, [75 ,25])
        iqr      = q75 - q25

        features_list.append([file_name, nobs, mean, skew, kurtosis, median, mode, std, low, peak, q25, q75, iqr, gender])
        print(f"\r{count}/{len(audio_files)}", end='')

        return
# #Calculo de tempo de disparo
start_time = time.time()

#inicio do Loop
loop = asyncio.get_event_loop()

#Controle de requisições por vez
sem = asyncio.Semaphore(600)

#Array de tasks
sents = []

nest_asyncio.apply()

#Coleta as recomendações para envio
gender_list = []
file_list = []
features_list = []

audio_files = os.listdir(source)
for k, file in enumerate(audio_files):
        if file.endswith('.wav'):       
                sent = asyncio.ensure_future(get_features(count=k+1, file=file))
                sents.append(sent)
 
done, _ = loop.run_until_complete(asyncio.wait(sents))
dataframe_features = pd.DataFrame(features_list, columns = ['FileName', 'nobs', 'mean', 'skew', 'kurtosis', 'median', 'mode', 'std', 'low', 'peak', 'q25', 'q75', 'iqr', 'Gender'])
dataframe_features.to_csv('D:\dev\Speaker-Gender-Recognition\data\MLS_split\Features_data.csv', index=False)<jupyter_output><empty_output><jupyter_text>Extract MFCCs  Get Metadata<jupyter_code>project_root =  os.path.dirname(os.path.dirname(os.getcwd()))
source = os.path.join(project_root, "MLS", "Full_split")

metadata = pd.read_csv(os.path.join(source, "metainfo.csv"))
data = metadata[['SPEAKER','GENDER']]
data_dict = dict(zip(data.SPEAKER, data.GENDER))<jupyter_output><empty_output><jupyter_text>Extract Features<jupyter_code>async def extract_MFCCs(count, file):
    async with sem:

        file_path = os.path.join(source, file)
        audio_data, sample_rate = librosa.load(file_path)
        mfccs = librosa.feature.mfcc(y=audio_data, sr=sample_rate)
        mfccs_mean = list(np.mean(mfccs.T, axis= 0))

        splited_file = file.split('_')
        if data_dict[int(splited_file[0])] == 'F':
            gender = 0
        if data_dict[int(splited_file[0])] == 'M': 
            gender = 1
        
        audio_data = parselmouth.Sound(file_path)
        audio_data = audio_data.values[0]
        sample_rate = 22050
        
        mfccs = librosa.feature.mfcc(y=audio_data, sr=sample_rate)
        
        mfccs_mean = list(np.mean(mfccs.T, axis= 0))
        
        sample_features = mfccs_mean
        sample_features.insert(0,str(file))
        sample_features.append(gender)
        
        print(f"\r{count}/{len(audio_files)}",end='')
        features_list.append(sample_features)
    
        return 
# #Calculo de tempo de disparo
start_time = time.time()

#inicio do Loop
loop = asyncio.get_event_loop()

#Controle de requisições por vez
sem = asyncio.Semaphore(600)

#Array de tasks
sents = []

nest_asyncio.apply()

#Coleta as recomendações para envio
gender_list = []
file_list = []
features_list = []

audio_files = os.listdir(source)
for k, file in enumerate(audio_files):
        if file.endswith('.wav'):
                sent = asyncio.ensure_future(extract_MFCCs(count=k+1, file=file))
                sents.append(sent)
 
done, _ = loop.run_until_complete(asyncio.wait(sents))

dataframe_features = pd.DataFrame(features_list, columns = ['FileName','MFCC_1','MFCC_2','MFCC_3','MFCC_4','MFCC_5',
                                                            'MFCC_6','MFCC_7','MFCC_8','MFCC_9','MFCC_10','MFCC_11',
                                                            'MFCC_12','MFCC_13','MFCC_14','MFCC_15','MFCC_16','MFCC_17',
                                                            'MFCC_18','MFCC_19','MFCC_20','Gender'])
dataframe_features.to_csv('D:\dev\Speaker-Gender-Recognition\data\MLS_split\MFCCs_data.csv', index=False)<jupyter_output><empty_output><jupyter_text>Extract f0  Get Metadata<jupyter_code>project_root =  os.path.dirname(os.path.dirname(os.getcwd()))
source = os.path.join(project_root, "MLS", "Full_split")

metadata = pd.read_csv(os.path.join(source, "metainfo.csv"))
data = metadata[['SPEAKER','GENDER']]
data_dict = dict(zip(data.SPEAKER, data.GENDER))<jupyter_output><empty_output><jupyter_text>Extract Features<jupyter_code>async def extract_F0(count, file):
    async with sem:
        file_path = os.path.join(source, file)
        
        audio_data = parselmouth.Sound(file_path)
        pitch = audio_data.to_pitch()
        pitch_values = pitch.selected_array['frequency']
        

        nobs_pitch, minmax_pitch, mean_pitch, variance_pitch, skew_pitch, kurtosis_pitch =  stats.describe(pitch_values)
        median_pitch   = np.median(pitch_values)
        mode_pitch     = stats.mode(pitch_values).mode[0]
        std_pitch      = np.std(pitch_values)
        low_pitch,peak_pitch = minmax_pitch
        q75_pitch,q25_pitch  = np.percentile(pitch_values, [75 ,25])
        iqr_pitch      = q75_pitch - q25_pitch
        
        splited_file = file.split('_')
        if data_dict[int(splited_file[0])] == 'F':
            gender = 0
        
        if data_dict[int(splited_file[0])] == 'M': 
            gender = 1
        
        
        sample_features = [nobs_pitch, mean_pitch, skew_pitch, kurtosis_pitch, median_pitch, mode_pitch, std_pitch, low_pitch, peak_pitch, q25_pitch, q75_pitch, iqr_pitch]
        sample_features.insert(0,str(file))
        sample_features.append(gender)
        
        string = ','.join(str(item) for item in sample_features)
        print(f"\r{count}/{len(audio_files)}",end='')
        features_list.append(sample_features)
    
        return 
# #Calculo de tempo de disparo
start_time = time.time()

#inicio do Loop
loop = asyncio.get_event_loop()

#Controle de requisições por vez
sem = asyncio.Semaphore(600)

#Array de tasks
sents = []

nest_asyncio.apply()

#Coleta as recomendações para envio
gender_list = []
file_list = []
features_list = []

audio_files = os.listdir(source)

for k, file in enumerate(audio_files):
    if file.endswith('.wav'):
        sent = asyncio.ensure_future(extract_F0(count=k+1, file=file))
        sents.append(sent)
 
done, _ = loop.run_until_complete(asyncio.wait(sents))

dataframe_features = pd.DataFrame(features_list, columns = ['FileName', 'nobs_pitch', 'mean_pitch', 'skew_pitch', 'kurtosis_pitch',
 'median_pitch', 'mode_pitch', 'std_pitch', 'low_pitch', 'peak_pitch', 'q25_pitch', 'q75_pitch', 'iqr_pitch', 'Gender'])

dataframe_features.to_csv('D:\dev\Speaker-Gender-Recognition\data\MLS_split\F0_data.csv', index=False)<jupyter_output>2336/4913
{'fraction_non_alphanumeric': 0.12834912834912834, 'fraction_numerical': 0.015561015561015561, 'mean_word_length': 3.5036880927291887, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '629902', 'license': 'MIT', 'n_tokens_mistral': 3181, 'n_tokens_neox': 2997, 'n_words': 523, 'path': 'notebooks/MLS_split/MLS_split_features_extractor.ipynb', 'repo_name': 'BrunoGianesi/Speker-Gender-Recognition', 'size': '8547'}