<jupyter_start><jupyter_text>Load audio files into dataset<jupyter_code># load (fe)male audio sequences without alpha, beta 
PATH = '../../datasets/freesound.org/wav_denoised/'
SR = 22050 

dataset = []

files = os.listdir(PATH)

for f in tqdm(files):
    if f.split('.')[-1] != 'wav':
        continue
    #pdb.set_trace()
    audio, sr = utils.audio.loadAudio(PATH + f, sr=SR)
    dataset.append(audio)
dataset = np.array(dataset)
len(dataset[0])
# read data
data = utils.misc.slidingWindow(dataset, int(SR*1), int(SR*1))
# create chunks
dataset_chunks = []
chunk_sz = int(SR*.2)
for chunk in dataset:
    #pdb.set_trace()
    dataset_chunks.append(utils.misc.slidingWindow(chunk, chunk_sz, chunk_sz))

# concatenate chunks
chunks = np.array(dataset_chunks)
chunks = np.concatenate((chunks[:]), axis=0)
chunks.shape
# mfcc 
#inp = chunks[np.random.randint(1000), np.random.randint(10)]
inp = chunks[np.random.randint(1000), :]
mfccs = librosa.feature.mfcc(y=inp, sr=SR, n_mfcc=20) #has quite a bunch of parameters
librosa.display.specshow(mfccs, sr=SR, x_axis='time')
ipd.Audio(inp, rate=SR)
#chroma
chroma_cq = librosa.feature.chroma_cqt(y=inp, sr=SR, hop_length=512)

#y=None, sr=22050, C=None, hop_length=512, fmin=None, norm=inf, threshold=0.0, tuning=None, n_chroma=12, n_octaves=7, window=None, bins_per_octave=36, cqt_mode='full')[source]

print(chroma_cq.shape)
librosa.display.specshow(chroma_cq, sr=SR, x_axis='time')
# RMS mag spec
print(librosa.feature.rms(y=inp).shape)

# Spectral centroid
print(librosa.feature.spectral_centroid(y=inp, sr=SR).shape)

# Spectral bandwidth
print(librosa.feature.spectral_bandwidth(y=inp, sr=SR).shape)

# Spectral flatness
print(librosa.feature.spectral_flatness(y=inp).shape)

# spectral rolloff
print(librosa.feature.spectral_rolloff(y=inp, sr=SR).shape)

# zero crossing rate 
print(librosa.feature.zero_crossing_rate(inp).shape)
# Spectral contrast

S = np.abs(librosa.stft(inp))
spec_contrast = librosa.feature.spectral_contrast(S=S, sr=sr)
print(spec_contrast.shape)
librosa.display.specshow(librosa.amplitude_to_db(S, ref=np.max), y_axis='log', x_axis='time')
# Tonnetz
y = librosa.effects.harmonic(inp)
tonnetz = librosa.feature.tonnetz(y=y, sr=SR)
print(tonnetz.shape)
# local and global tempogram
hop_length = 512
oenv = librosa.onset.onset_strength(y=inp, sr=SR, hop_length=hop_length)
tempogram = librosa.feature.tempogram(onset_envelope=oenv, sr=SR,
                                      hop_length=hop_length)
# Compute global onset autocorrelation
ac_global = librosa.autocorrelate(oenv, max_size=tempogram.shape[0])
ac_global = librosa.util.normalize(ac_global)
# Estimate the global tempo for display purposes
tempo = librosa.beat.tempo(onset_envelope=oenv, sr=sr,
                           hop_length=hop_length)[0]

from skimage.transform import resize
bottle_resized = resize(tempogram, (32, 44))
print(tempogram.shape)
print(bottle_resized.shape)
#librosa.display.specshow(tempogram, x_axis='time')
librosa.display.specshow(bottle_resized, x_axis='time')
from skimage.transform import resize

num_points = 1000
num_feats = 20
num_chunks = chunks.shape[0]
hop_length = 512
X = np.zeros((num_chunks, 44*(13 + 12 + 1 + 1 + 1 + 1 + 1 + 1 + 7 + 6 + 32) + 1)) # 13 mfcc, 12 chroma, 
for i in tqdm(range(num_chunks)):
    #pdb.settrace()
    # mfcc
    feat = librosa.feature.mfcc(y=chunks[i, :], sr=SR, n_mfcc=num_feats)[1:14].flatten() # 13x44
    # chroma
    chroma_cq = librosa.feature.chroma_cqt(y=chunks[i, :], sr=SR, hop_length=hop_length).flatten() #12x44
    # RMS mag spec
    rms = librosa.feature.rms(y=chunks[i, :]).flatten() #1x44
    # Spectral centroid
    cent = librosa.feature.spectral_centroid(y=chunks[i, :], sr=SR).flatten()#1x44
    # Spectral bandwidth
    band = librosa.feature.spectral_bandwidth(y=chunks[i, :], sr=SR).flatten()#1x44
    # Spectral flatness
    flat = librosa.feature.spectral_flatness(y=chunks[i, :]).flatten()#1x44
    # spectral rolloff
    rolloff = librosa.feature.spectral_rolloff(y=chunks[i, :], sr=SR).flatten()#1x44
    # zero crossing rate 
    zcross = librosa.feature.zero_crossing_rate(chunks[i, :]).flatten()#1x44
    # Spectral contrast
    S = np.abs(librosa.stft(chunks[i, :]))
    spec_contrast = librosa.feature.spectral_contrast(S=S, sr=SR).flatten() #7x44
    # Tonnetz
    y = librosa.effects.harmonic(chunks[i, :])
    tonnetz = librosa.feature.tonnetz(y=y, sr=SR).flatten() #6x44
    # local and global tempogram    
    oenv = librosa.onset.onset_strength(y=chunks[i, :], sr=SR, hop_length=hop_length)
    tempogram = resize(librosa.feature.tempogram(onset_envelope=oenv, sr=SR, hop_length=hop_length), (32, 44)) #32x44
    # Compute global onset autocorrelation
    ac_global = librosa.autocorrelate(oenv, max_size=tempogram.shape[0])
    ac_global = librosa.util.normalize(ac_global)
    # Estimate the global tempo for display purposes
    tempo = librosa.beat.tempo(onset_envelope=oenv, sr=sr, hop_length=hop_length)[0]
    #pdb.set_trace()
    X[i, :] = np.concatenate((feat, chroma_cq, rms, cent, band, flat, rolloff, zcross, spec_contrast, tonnetz, tempogram.flatten(), tempo.reshape(1)))
np.save("feature_space.npy", X)
X = np.load("feature_space.npy")
X.shape
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler

new_X = StandardScaler().fit_transform(X)

new_X = X

pca = PCA(n_components=128)
#pca_X = pca.fit_transform(new_X)

pca_X = new_X
from sklearn.mixture import GaussianMixture
gmm = GaussianMixture(n_components=64, verbose=2).fit(pca_X)
labels = gmm.predict(pca_X)
print(labels.shape)
plt.hist(labels)
#print(labels[1000:2000])
#%matplotlib inline
#pca_scaled_X = StandardScaler().fit_transform(pca_X)
pca_scaled_X = pca_X
plot_X = PCA(n_components=3).fit_transform(pca_scaled_X)
%matplotlib inline
colors = ['r', 'b', 'g', 'm', 'k', 'c']
fig1 = plt.figure(figsize=[20, 10])
ax1 = fig1.add_subplot(111, projection='3d')
for i in range(64):
    ax1.scatter(plot_X[labels==i, 0], plot_X[labels==i, 1], plot_X[labels==i, 2], c=colors[i%6], s=5.);
plt.show()
#%matplotlib inline
plot_X2 = PCA(n_components=2).fit_transform(pca_scaled_X)
colors = ['red', 'blue', 'green', 'magenta', 'black', 'darkgreen', 'orange', 'crimson', 'grey'] #'cyan',
fig2 = plt.figure(figsize=[10, 5])
ax2 = fig2.add_subplot(111)
for i in range(64):.-+
    ////////////////////////////////////////////////////////////////////
    ax2.scatter(plot_X2[labels==i, 0], plot_X2[labels==i, 1], c=colors[i%6], s=5);
plt.show()
#plt.savefig('mfcc_spectral_features.png')
#%matplotlib inline
plot_X2 = PCA(n_components=2).fit_transform(pca_X)
colors = ['red', 'blue', 'green', 'magenta', 'black', 'darkgreen', 'orange', 'crimson', 'grey'] #'cyan',
fig2 = plt.figure(figsize=[10, 5])
ax2 = fig2.add_subplot(111)
for i in range(6):
    ax2.scatter(plot_X2[labels==i, 0], plot_X2[labels==i, 1], c=colors[i], s=5);
plt.show()
target = 0
idx = np.where(labels==target)[0][0]
print(labels[idx])
#i = idx // num_chunks
#j = idx % num_chunks
display(ipd.Audio(chunks[idx, :], rate=SR))

idx = np.where(labels==target)[0][5]
display(ipd.Audio(chunks[idx, :], rate=SR))

idx = np.where(labels==target)[0][10]
display(ipd.Audio(chunks[idx, :], rate=SR))

idx = np.where(labels==target)[0][15]
display(ipd.Audio(chunks[idx, :], rate=SR))

idx = np.where(labels==target)[0][20]
display(ipd.Audio(chunks[idx, :], rate=SR))

idx = np.where(labels==target)[0][30]
display(ipd.Audio(chunks[idx, :], rate=SR))

idx = np.where(labels==target)[0][10]
print(labels[idx])
#i = idx // num_chunks
#j = idx % num_chunks
ipd.Audio(chunks[idx, :], rate=SR)
idx = np.where(labels==target)[0][20]
print(labels[idx])
#i = idx // num_chunks
#j = idx % num_chunks
ipd.Audio(chunks[idx, :], rate=SR)
idx = np.where(labels==target)[0][30]
print(labels[idx])
#i = idx // num_chunks
#j = idx % num_chunks
ipd.Audio(chunks[idx, :], rate=SR)<jupyter_output>1
{'fraction_non_alphanumeric': 0.17263719512195122, 'fraction_numerical': 0.028709349593495935, 'mean_word_length': 5.258346581875994, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '426793', 'license': 'MIT', 'n_tokens_mistral': 3359, 'n_tokens_neox': 3163, 'n_words': 541, 'path': 'synthesizer/full_audio/clustering_mfcc-freesound-Copy1.ipynb', 'repo_name': 'Virtual-Audience-Synthesis/audience-synthesis-playground', 'size': '7872'}