<jupyter_start><jupyter_text>Imports and enviroment setup<jupyter_code>import pandas as pd
import numpy as np
import os

import matplotlib.pyplot as plt

import sklearn
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder, StandardScaler
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
from sklearn.neighbors import NearestNeighbors, BallTree
from sklearn.decomposition import TruncatedSVD
from sklearn.manifold import TSNE


import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, RNN
from keras.layers.embeddings import Embedding
from keras.preprocessing import sequence

# random state
rs = 42<jupyter_output><empty_output><jupyter_text>Load and clean data<jupyter_code># load data
# URL = 'https://github.com/Build-Week-Track-Team-7/explore/blob/main/data.csv.zip?raw=true'

# path_to_zip = tf.keras.utils.get_file('./data.zip', origin=URL, extract=True)
# PATH = os.path.join(os.path.dirname(path_to_zip), 'data')

df_raw = pd.read_csv('./data.csv')
df_raw.head()

    
def clean_data(df):
  '''
  a functions that drops columns which either 
  add bias or noise to dataset 

  raw text data
  ---
  input - dataframe object/ndarray
  output - dataframe object/ndarray
  '''
  # bias
  df_cleaned = df.drop('artists', axis=1)
    # duplicate/triplicate information may disproportantly weight results
  df_cleaned = df_cleaned.drop('release_date', axis=1)
    # duplicate/triplicate information may disproportantly weight results

  # noise
  df_cleaned = df_cleaned.drop('id', axis=1)
    # has no meaningful contribution to results

  # drop rows with missing values
  df_cleaned = df_cleaned.dropna()

  # cast column(s) to integer dtype
  # df_cleaned = df_cleaned.astype({'acousticness': 'int64',
  #                                 'danceability': 'int64',
  #                                 'energy': 'int64',
  #                                 'instrumentalness': 'int64',
  #                                 'liveness': 'int64',
  #                                 'liveness': 'int64',
  #                                 'loudness': 'int64',
  #                                 'speechiness': 'int64',
  #                                 'tempo': 'int64',
  #                                 'valence': 'int64'}).dtypes

  return df_cleaned


data_cleaned = clean_data(df_raw)
print(data_cleaned.shape)
data_cleaned.head()
# shuffle dataset to avoid bias
num_rows = 169909
data_cleaned = data_cleaned.sample(n=num_rows, 
                                   replace=False,
                                   random_state=rs
                                   )

# reindex 
data_cleaned = data_cleaned.reset_index(drop=True)

data_cleaned.head()<jupyter_output><empty_output><jupyter_text>EDA<jupyter_code>data_cleaned.describe()
data_cleaned.describe(exclude='number')
data_cleaned.info()
# scatter plot of features as they relate to sampling of individual songs
# features within the -25 to 0 range
plt.scatter(data_cleaned['loudness'][:20], data_cleaned['name'][:20], c='navy')
plt.show()

# features within the 0-1 range
plt.scatter(data_cleaned['acousticness'][:20], data_cleaned['name'][:20], c='red')
plt.scatter(data_cleaned['danceability'][:20], data_cleaned['name'][:20], c='blue')
plt.scatter(data_cleaned['energy'][:20], data_cleaned['name'][:20], c='orange')
plt.scatter(data_cleaned['liveness'][:20], data_cleaned['name'][:20], c='lightgreen')
plt.scatter(data_cleaned['valence'][:20], data_cleaned['name'][:20], c='tomato')

plt.show()
# features that may have some correlation
plt.scatter(data_cleaned['instrumentalness'][:20], data_cleaned['name'][:20], c='pink')
plt.scatter(data_cleaned['speechiness'][:20], data_cleaned['name'][:20], c='plum')
plt.show()
# boolean values
plt.scatter(data_cleaned['explicit'][:20], data_cleaned['name'][:20], c='yellow')
plt.scatter(data_cleaned['mode'][:20], data_cleaned['name'][:20], c='peachpuff')
plt.show()
# features within the 1 - 11 range
plt.scatter(data_cleaned['key'][:20], data_cleaned['name'][:20], c='gray')
plt.show()
# features within the 70 - 170 range
plt.scatter(data_cleaned['tempo'][:20], data_cleaned['name'][:20], c='goldenrod')
plt.show()
# features within years 1921 - 2020 
plt.scatter(data_cleaned['year'][:20], data_cleaned['name'][:20], c='blueviolet')
plt.show()
# features within the 200000 - 800000 range
plt.scatter(data_cleaned['duration_ms'][:20], data_cleaned['name'][:20], c='green')
plt.show()<jupyter_output><empty_output><jupyter_text>Additional preproccessing of dataPipeline 1 - (instantiate, fit_transform)*   StandardScalerPipeline 2a, 2b, 2c, and 2d - (instantiate, fit, predict)2a:*   K-NN - (exploring in isolation for similiarity clustering efficienies)2b:*   K-NN - (exploring for similiarity clustering efficienies)*   PCA - (exploring for efficiency boost not dimensionality reduction b/c this is not needed; already done by limiting features)2c:*   K-NN - (exploring for similiarity clustering efficienies)*   t_SNE - (exploring for additional efficiencies in dimensionality reduction)2d:*   t_SNE - (exploring in isolation for efficiencies in dimensionality reduction)  Pipeline 1*   StandardScaler<jupyter_code># create feature matrix
X = data_cleaned.drop('name', axis=1)
y = data_cleaned['name']


# normalize standardize data using pipeline (instantiate, fit, transform)
# onehotender, ordinal encoder, standard scaler in order

# instantiate pipeline
pipe_1 = Pipeline([
                 #('onehotencoder', OneHotEncoder()), 
                 #('ordinalencoder', OrdinalEncoder()), 
                 ('scaler', StandardScaler()), 
                 ])

# fit data to transformer and transform data
pipe_1.fit_transform(X, y=None)


#  check data
print(X.shape)
print(y.shape)

# split data - train/test split
X_train, X_test, y_train, y_test = train_test_split(X, y, 
                                                    test_size=0.20, 
                                                    random_state=42
                                                    )

# X_arr = np.asarray(X)
# y_arr = np.asarray(y)


# # split data - train/test split
# X_train, X_test, y_train, y_test = train_test_split(X_arr, y_arr, 
#                                                     test_size=0.20, 
#                                                     random_state=42
#                                                     )

# check data
print(X_train.shape)
print(y_train.shape)
print(X_test.shape)
print(y_test.shape)<jupyter_output>(135927, 15)
(135927,)
(33982, 15)
(33982,)<jupyter_text>Pipeline *2a*2a:K-NN - (exploring in isolation for similiarity clustering efficienies)<jupyter_code># hyperparameters for piplines
# K-NN estimator
n_neigh = 25
  # must be an interger or will throw an error
alg = 'ball_tree'

#TFIDF Vectorizer
max_df=5
max_fea=10000
min_df=1
st_wds='english'
use_idf=True

# SVD
n_comp=14
  # must be an integer
_alg='arpack'
  # as an eigensolver for greater efficiency while maintaining accuracy when interacing with Spotify API
  # can switch to 'randomized' for greater efficiency if needed

# KMeans
n_clus=10
init='k-means++'
max_iter=100
n_init=1
verbose=2



# instantiate model
nbrs = NearestNeighbors(n_neighbors=n_neigh, 
                 algorithm=alg)

# fit data
nbrs = nbrs.fit(X_train)


# check data
print(X_train.shape)
print(y_train.shape)
print(X_test.shape)
print(y_test.shape)

#dir(nbrs)<jupyter_output><empty_output><jupyter_text>Pipeline *2b*2b:Tfidf - (vectorizer)t-SVD - (exploring for efficiency boost not dimensionality reduction b/c this is not needed; already done by limiting features)KMeans - (exploring similarities through clustering)<jupyter_code># instantiate pipeline
pipe_2b = Pipeline([
                 ('tfidf', TfidfVectorizer(max_df=max_df, 
                                           max_features=max_fea,
                                           min_df=min_df,
                                           stop_words=st_wds,
                                           use_idf=True)
                 ),
                 ('svd', TruncatedSVD(n_components=n_comp,
                             algorithm=_alg)
                 ),
                 ('km', KMeans(n_clusters=n_clus,
                                   init=init,
                                   max_iter=max_iter,
                                   n_init=n_init,
                                   verbose=verbose)
                 ), 
                 ])

# fit data to transformer and transform data
km = pipe_2b.fit(X_train)


# check data
print()
print(X_train.shape)
print(y_train.shape)
print(X_test.shape)
print(y_test.shape)<jupyter_output><empty_output><jupyter_text>Pipeline *2c*2c:K-NN - (exploring for similiarity clustering efficienies)t_SNE - (exploring for additional efficiencies in dimensionality reduction)<jupyter_code><jupyter_output><empty_output><jupyter_text>Pipeline *2d*2d:t_SNE - (exploring for additional efficiencies in dimensionality reduction)<jupyter_code><jupyter_output><empty_output><jupyter_text>Neural Network model(s) NN 1 - (instantiate, compile, fit)*   CNN> Indented blockNN 2 - (instantiate, compile, fit)*   LSTMNN 3 - (instantiate, compile, fit)*   RNN (deep and wide)  NN 1 - (instantiate, compile, fit)CNN<jupyter_code># define and tune hyperparameters within NN
# CNN
batch_size = 32
epochs = 5


# LSTM layer(s)
units_lstm = len(np.unique(y_train)) * .75
units_lstm = units_lstm.is_integer()
do = 0.2

# Embedding layer
max_feat = 10000
units_em = len(np.unique(y_train)) * .75
units_em = units_em.is_integer()

# Dense hidden layer(s)
act_hid = 'relu'

# Dense output layer
units_out = len(np.unique(y_train))
act_out = 'softmax'
  # for use with multiclass

# general use
maxlen = 80
shape = X_train.shape[1]

# compile and fit
loss='sparse_categorical_crossentropy'
batch_sz = 32
opt = 'nadam'
metric='accuracy'

X_train = np.array([np.expand_dims(x_, 1) for x_ in X_train.values])
X_test = np.array([np.expand_dims(x_, 1) for x_ in X_test.values])
# check data
print()
print(X_train.shape)
print(y_train.shape)
print(X_test.shape)
print(y_test.shape)
# of lables in output layer
len(np.unique(y_train))
# Setup Architecture

# instantiate model
model = Sequential()

# -- preprocessing portion of CNN network --

# hidden pre-processing layer 1 with implicit input layer
# creates feature map
# activation function used to produce updated weights (allows backpropegation)
model.add(Conv2D(32, (3,3), activation='relu', input_shape=(32,32, 15)))

# hidden preprocessing layer 2
# subsample values
model.add(MaxPooling2D((2,2)))

# hidden preprocessing layer 3
# creates feature map
# activation function used to produce updated weights (allows backpropegation)
model.add(Conv2D(64, (3,3), activation='relu'))

# hidden preprocessing layer 4
# subsample values
model.add(MaxPooling2D((2,2)))

# hidden preporocessing layer 5
# creates feature map
# activation function used to produce updated weights (allows backpropegation)
model.add(Conv2D(64, (3,3), activation='relu'))

# hidden preporcessing layer 6
# standardiztion/normalization 
model.add(GlobalAveragePooling2D())
  # a Fancy version of Flatten with subsample values

# -- fully connected portion of CNN network --

# hidden layer 7
model.add(Dense(64, activation='relu'))

# output layer 8
model.add(Dense(10, activation='softmax'))

model.summary()
# Compile Model

model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])
results = model.fit(
    train_data_gen,
    steps_per_epoch=total_train // batch_size,
    epochs=epochs,
    validation_data=val_data_gen,
    validation_steps=total_val // batch_size
)<jupyter_output><empty_output><jupyter_text>NN 2 - (instantiate, compile, fit)LSTM<jupyter_code># check data
print()
print(X_train.shape)
print(y_train.shape)
print(X_test.shape)
print(y_test.shape)
# of lables in output layer
len(np.unique(y_train))
## TODO...tokenize lemmas
# instantiating a Sequential class 
model = Sequential()

# explicit input layer  
#model.add(Embedding(max_feat, units_em))

# hidden layer 1 
model.add(LSTM(units=units_lstm, dropout=do, return_sequences=True, input_shape=shape))

# hidden layer 2
#model.add(LSTM(units=units_lstm, dropout=do, return_sequences=True))

# hidden layer 3 
#model.add(LSTM(units=units_lstm, dropout=do))

# hidden layer 4
#model.add(Dense(units=units_out, activation=act_hid))

# output layer 
model.add(Dense(units_out, activation=act_out))

model.compile(loss=loss,
              optimizer=opt, 
              metrics=[metric])

# fit model
results = model.fit(X_train, y_train,
          batch_size=32, 
          epochs=5,
          steps_per_epoch=1000, 
          validation_data=(X_test, y_test)
)

# display model
model.summary()
import matplotlib.pyplot as plt

# Plot training & validation loss values
plt.plot(results_nn2.history['loss'])
plt.plot(results_2.history['val_loss'])
plt.title('Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper left')
plt.show();
import matplotlib.pyplot as plt

# Plot training & validation loss values
plt.plot(results_nn2.history['accuracy'])
plt.plot(results_2.history['val_accuracy'])
plt.title('Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper left')
plt.show();<jupyter_output><empty_output><jupyter_text>NN 3 - (instantiate, compile, fit)RNN (deep and wide)<jupyter_code><jupyter_output><empty_output><jupyter_text>Evaluate model<jupyter_code><jupyter_output><empty_output><jupyter_text>Vizualize results<jupyter_code><jupyter_output><empty_output><jupyter_text>Conclusions<jupyter_code><jupyter_output><empty_output><jupyter_text>Export to application<jupyter_code># convert to json<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11633237822349571, 'fraction_numerical': 0.02005730659025788, 'mean_word_length': 3.6182600066159445, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 61, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '370779', 'license': 'MIT', 'n_tokens_mistral': 4837, 'n_tokens_neox': 4508, 'n_words': 1009, 'path': 'Spotify_recommender_NN_explore_RJProctor.ipynb', 'repo_name': 'Build-Week-Track-Team-7/explore', 'size': '13960'}