<jupyter_start><jupyter_text>Wilson's Morning Wake Up Playlist Generator, Feature EngineeringThis notebook is broken down into the following tasks:* Clean and pre-process the data.* Standardization and normalization of numerical variables* Define features for harmonic sequencing* Create train/test `.csv` files that hold the relevant features and class labels for train/test data pointsIn the _next_ notebook, Train_Deploy , we will use the features and `.csv` files created in _this_ notebook to train an estimator locally and on AWS SageMaker.<jupyter_code># Import librariess
import pandas as pd
import numpy as np
import os
import pickle
from pickle import dump
import joblib
import random<jupyter_output><empty_output><jupyter_text>This Wilson Morning Wake Up dataset is summarized in a `.csv` file named `wmw_tracks.csv`, which we can read in using `pandas`. This was created in the Explore notebook.<jupyter_code># Read file
csv_file = 'data/wmw_tracks.csv'
wmw_df = pd.read_csv(csv_file)
wmw_df.head()
# Show duplicated songs - woops!
wmw_df[wmw_df.duplicated(['track_name', 'artist_name'], keep=False)].sort_values('track_name')
wmw_df.columns
# HOLD ON to duplicates
# print("Initial shape:", wmw_df.shape[0], 'tracks')
# wmw_df.drop_duplicates(subset=['track_name', 'artist_name'], inplace=True)
# print("New shape:", wmw_df.shape[0], 'tracks')
# Exclude unnecessary columns
exclude_cols = ['track_name', 'artist_name', 'duration_ms', 'track_href', 'uri', 'time_signature', 'id', 'type', 'analysis_url', 'mode','key']

wmw_df.drop(columns=exclude_cols, inplace=True)
# Feature list
feature_list = ['danceability','energy', 'loudness', 'speechiness', 'acousticness', 'instrumentalness', 'liveness', 'valence', 'tempo']

# Hold out: 'mode','key','tempo'

# Output list
output_list = ['danceability','energy', 'loudness', 'speechiness', 'acousticness', 'instrumentalness', 'liveness', 'valence', 'tempo']<jupyter_output><empty_output><jupyter_text>--- Standardise features<jupyter_code>from sklearn.preprocessing import StandardScaler

# Standardize features
standard_scaler = StandardScaler()

standard_scaler.fit(wmw_df[feature_list])

standard_features = standard_scaler.transform(wmw_df[feature_list])

processed_df = pd.DataFrame(standard_features, index=wmw_df.index, columns=wmw_df.columns[2:])

wmw_df = pd.concat([wmw_df[['volume', 'position']],processed_df[feature_list]], axis=1)

wmw_df.head(10)
wmw_df.shape
# Save the scaler
# dump(standard_scaler, open('utilities/standard_features.pkl', 'wb'))<jupyter_output><empty_output><jupyter_text>--- Fit PCA for track selection in post-processing<jupyter_code>from sklearn.decomposition import PCA

pca = PCA(n_components=3)

pca.fit(wmw_df[feature_list])

print(pca.explained_variance_ratio_)

print(pca.singular_values_)
pca_df = pca.transform(wmw_df[feature_list])
import matplotlib.pyplot as plt, numpy as np
from mpl_toolkits.mplot3d import Axes3D

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(pca_df[:,0], pca_df[:,1], pca_df[:,2])
plt.show()
# Save PCA
# dump(pca, open('artefacts/dim_red.pkl', 'wb'))<jupyter_output><empty_output><jupyter_text>--- Prepare inputs and target features for training Given the playlists variable-length sequences, it will only take the first 12 tracks of each volume. In the future, the model will account for variable-length inputs using padding.<jupyter_code># Prepare labels using shift
training_df = pd.DataFrame()

for i in wmw_df['volume'].unique():
    X_df = wmw_df[wmw_df['volume'] == i].iloc[:13]
    y_df = wmw_df[wmw_df['volume'] == i].iloc[:13].shift(-1)[output_list]
    X_y_df = pd.concat([X_df, y_df.add_prefix('y_')], axis=1).dropna()
    training_df = training_df.append(X_y_df, ignore_index=True)
    
    # Clean up
    del X_df
    del y_df
    del X_y_df
training_df.shape
training_df.columns
training_df.head()<jupyter_output><empty_output><jupyter_text>--- Save tensor friendly data for Deep Learning<jupyter_code>import torch

tracks = torch.from_numpy(training_df[output_list].values)
tracks.shape
tracks.shape, tracks.stride()
playlist_tracks = tracks.view(-1, 12, tracks.shape[1])
playlist_tracks.shape, playlist_tracks.stride()
training_df.to_csv("data/tensor_train.csv", index=False)<jupyter_output><empty_output><jupyter_text>--- Quick look at the results<jupyter_code>%matplotlib inline

import seaborn as sns
import matplotlib.pyplot as plt

plt.figure(figsize=(20,5))

ax = sns.countplot(x="volume", data=training_df)

# training_df.plot.bar(x='volume', rot=0)<jupyter_output><empty_output><jupyter_text>--- Save data for traditional Machine Learning Not of use for now.<jupyter_code># def make_csv(x, filename, data_dir):
#     '''Merges features and labels and converts them into one csv file with labels in the first column.
#        :param x: Data features
#        :param y: Data labels
#        :param file_name: Name of csv file, ex. 'train.csv'
#        :param data_dir: The directory where files will be saved
#        '''
#     # make data dir, if it does not exist
#     if not os.path.exists(data_dir):
#         os.makedirs(data_dir)
    
#     data = pd.DataFrame(x).dropna()
#     data.to_csv(os.path.join(data_dir, filename), index=False)
    
#     # nothing is returned, but a print statement indicates that the function has run
#     print('Path created: '+str(data_dir)+'/'+str(filename))
# # Can change directory, if you want
# data_dir = 'data'

# make_csv(wmw_df, filename='wmw.csv', data_dir=data_dir)
# # Can change directory, if you want
# data_dir = 'data'

# make_csv(training_df, filename='train.csv', data_dir=data_dir)
# training_df.iloc[:, 2:10].head()
# training_df.iloc[:, 11:].head()
# # Build sequences and targets
# def create_playlist_sequences(input_data):
#     input_playlists = []
    
#     for i in input_data['volume'].unique():
#         temp_vol = input_data[input_data['volume'] == i]
#         X = temp_vol.iloc[:, 2:10].values
#         y = temp_vol.iloc[:, 10:].values
#         input_playlists.append((X, y))
        
#     return input_playlists
# # Gather sequences and targets
# processed_data = create_playlist_sequences(training_df)

# # Take a look at first input and target
# print("X", processed_data[0][0][0])
# print("y", processed_data[0][1][0])
# from unittest.mock import MagicMock, patch

# def _print_success_message():
#     print('Tests Passed!')

# def test_playlist_sequences(input_playlists):
    
#     track_features = [-2.39099487, -2.63509459, -0.27732204,  0.92969533, -0.48983686,-1.15691947,  1.08569029, -1.20454903,  2.09618458]
    
#     track_features_len = 9
#     target_features_len = 8
    
#     # check shape and equality of first track
#     assert len(input_playlists[0][0][0]) == len(track_features), \
#         'Number of features in input_playlist features does not match expected number of ' + str(len(track_features))    
    
#     # check shape of input and output arrays
#     assert input_playlists[0][0].shape[1]==track_features_len, \
#         'input_features should have as many columns as selected features, got: {}'.format(train_x.shape[1])
#     assert input_playlists[0][1].shape[1]==target_features_len, \
#         'target_features should have as many columns as selected features, got: {}'.format(train_x.shape[1])
    
#     #TODO: Add more tests
    
#     _print_success_message()
# Test processed sequences
# test_playlist_sequences(processed_data)<jupyter_output><empty_output><jupyter_text>--- Visualise processed features<jupyter_code># Read file
csv_file = 'data/wmw_tracks.csv'
df = pd.read_csv(csv_file)
df.head()
# Setup feature standardisation
std_scaler = joblib.load('artefacts/standard_features.pkl')

# Setup dimensionality reduction for track picking
dim_red = joblib.load('artefacts/dim_red.pkl')
# Prepare labels using shift
pca_df = pd.DataFrame()

for i in df['volume'].unique():
    X_df = df[df['volume'] == i].iloc[:13]
    y_df = df[df['volume'] == i].iloc[:13].shift(-1)[output_list]
    X_y_df = pd.concat([X_df, y_df.add_prefix('y_')], axis=1).dropna()
    pca_df = pca_df.append(X_y_df, ignore_index=True)
    
    # Clean up
    del X_df
    del y_df
    del X_y_df
# Features
track_features = ['danceability', 'energy', 'loudness', 'speechiness', 
                  'acousticness', 'instrumentalness', 'liveness', 'valence', 'tempo']

pca_df[track_features] = std_scaler.transform(pca_df[track_features])

coordinates = dim_red.transform(pca_df[track_features])
pca_df = pd.concat([pca_df[['position', 'volume', 'track_name']], pd.DataFrame(coordinates, columns=['one_comp', 'two_comp', 'third_comp'])], axis=1)
pca_df.describe()<jupyter_output><empty_output><jupyter_text>Analyse the variance of tracks at each track position<jupyter_code>from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import numpy as np

# Create figure outside the loop
fig = plt.figure(figsize=(15,15))

for i in range(1, len(pca_df['position'].unique().tolist()) + 1):

    # Add the i+1 subplot of the x.shape[0] x 1 grid
    ax = fig.add_subplot(4, 3, i, projection='3d')
    ax.set_title("Track Position: " + str(i))
    temp_df = pca_df[pca_df['position'] == i]
    ax.scatter(temp_df['one_comp'], temp_df['two_comp'], temp_df['third_comp'], c='r', marker='o')

    ax.set_xlabel('X')
    ax.set_ylabel('Y')
    ax.set_zlabel('Z')
    
    ax.set_xlim(-125, -116)
    ax.set_ylim(-13, -7)
    ax.set_zlim(-2, 3)
    
    
# Show it outside the loop
plt.show()
# Create figure outside the loop
fig = plt.figure(figsize=(15, 10))

# plot Economic Need Index vs School Income Estimate by group
groups = pca_df.groupby('position')

ax = fig.add_subplot(1, 1, 1, projection='3d')
# ax.set_title("Wilson Morning's Wake Up Tracks by Position")

for label, group in groups:
    ax.scatter(group['one_comp'], group['two_comp'], group['third_comp'], label=label)

ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Z')

ax.set_xlim(-125, -116)
ax.set_ylim(-13, -7)
ax.set_zlim(-2, 3)


ax.legend(title='Track Positions')

    
# Show it outside the loop
plt.show()
# Create figure outside the loop
fig = plt.figure(figsize=(15,15))

# random_volumes = random.randint(, 6)
random_volumes = random.sample(range(0, len(pca_df['volume'].unique().tolist())), 6)


for i in range(len(random_volumes)):

    # Add the i+1 subplot of the x.shape[0] x 1 grid
    ax = fig.add_subplot(3, 2, i+1, projection='3d')
    ax.set_title("WMW Volume: " + str(random_volumes[i]))
    temp_df = pca_df[pca_df['volume'] == random_volumes[i]]
    
    for i in range(len(temp_df)):
        x, y, z = temp_df.iloc[i]['one_comp'], temp_df.iloc[i]['two_comp'], temp_df.iloc[i]['third_comp']
        ax.scatter(x, y, z)
        ax.text(x, y, z, '{0}'.format(temp_df.iloc[i]['position']), size=12)
    
    ax.plot(temp_df['one_comp'], temp_df['two_comp'], temp_df['third_comp'])
    
    ax.set_xlabel('X')
    ax.set_ylabel('Y')
    ax.set_zlabel('Z')
    
    ax.set_xlim(-125, -116)
    ax.set_ylim(-13, -7)
    ax.set_zlim(-2, 3)
    
plt.tight_layout(pad=0.4, w_pad=0.5, h_pad=1.0)

# Show it outside the loop
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14095768574252088, 'fraction_numerical': 0.020393495642799388, 'mean_word_length': 4.921276595744681, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '583749', 'license': 'MIT', 'n_tokens_mistral': 4131, 'n_tokens_neox': 3960, 'n_words': 908, 'path': 'notebooks/2_Feature_Engineering.ipynb', 'repo_name': 'alistairwgillespie/wmw_deep_playlist_generator', 'size': '11131'}