<jupyter_start><jupyter_text>Imports and enviroment setup<jupyter_code>import pandas as pd
import numpy as np
import os
import seaborn as sns

import matplotlib.pyplot as plt

from scipy.stats import t, ttest_1samp, ttest_ind

import sklearn
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder, StandardScaler
from sklearn.neighbors import NearestNeighbors, KNeighborsClassifier
from sklearn.manifold import TSNE
from sklearn.pipeline import Pipeline

import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, RNN
from keras.layers.embeddings import Embedding
from keras.preprocessing import sequence
pd.set_option('display.max_rows',200)
pd.set_option('display.max_columns',100)
from sklearn.feature_extraction.text import TfidfVectorizer<jupyter_output><empty_output><jupyter_text>Load and clean data<jupyter_code># # load data
# URL = 'https://github.com/Build-Week-Track-Team-7/explore/blob/main/data.csv.zip?raw=true'

# path_to_zip = tf.keras.utils.get_file('data.zip', origin=URL, extract=True)
# PATH = os.path.join(os.path.dirname(path_to_zip), 'data')

data = pd.read_csv('data.csv')
data.head()

data.shape
list(data.columns)
# sometimes floats are indicators of NaNs present; we may want to look further 
# into that
data.dtypes
data_cleaned = data
data_cleaned.head()
data_cleaned['release_date'] = pd.to_datetime(data_cleaned['release_date'])
data_cleaned = data_cleaned.sort_values(by=['release_date'], ascending = False)
data_cleaned['year'] = pd.DatetimeIndex(data_cleaned['release_date']).year
print(data_cleaned.shape)
data_cleaned.head()
data_cleaned['year'].value_counts()
data_cleaned['key'].value_counts()
sns.relplot(x="year", y="popularity", data=data_cleaned, height=5, aspect=16/9)
list(data_cleaned.columns)
plt.scatter(data_cleaned['release_date'], data_cleaned['popularity'], c=data_cleaned['explicit'])
plt.show()
data_cleaned_key_cleanup = data.drop('artists', axis=1)
data_cleaned_key_cleanup = data_cleaned_key_cleanup.drop('id', axis=1)
print(data_cleaned_key_cleanup.shape)
data_cleaned_key_cleanup.head()
data_cleaned.describe()
# creating a dictionary to make the keys presentable as names rather than 
# numbers
cleanup = {'key':{0:'C',1:'CD',2:'D',3:'DE',4:'E',5:'EF',6:'F',7:'FG',8:'G',
                  9:'GA',10:'A',11:'AB'},
           'mode':{1:'Major',0:'Minor'},
           'explicit':{1:'Explicit',0:'Not Explicit'}}
data_cleaned_key_cleanup.replace(cleanup, inplace=True) #only run once
data_cleaned_key_cleanup.head(10)
key_data = data_cleaned_key_cleanup[['popularity','year','key']]
sns.pairplot(key_data, hue = 'key')
import plotly.express as px
pip install --upgrade plotly
data_cleaned_key_cleanup['key'].value_counts()
all = 169909
C=21499/all
FG=20757/all
D=18821/all
GA=17628/all
EF=16336/all
E=12921/all
CD=12816/all
A=12056/all
G=10711/all
AB=10593/all
F=8586/all
DE=7185/all
keys=[<KEY>]
list(keys)
keyavg = np.average(keys)
print(keyavg)
# we want a value that when added to the top 50 percentile of the relative sum 
# of keys the result is more positive (100%<) and adding to the bottom 50 
# percentile is more negative(100%>)
x1 = 1 - 0.12653243795208022
x2 = 1 - 0.04228734204780206
newkeyavg = (x1+x2)/2
print(newkeyavg)
keyindex = [i + newkeyavg for i in keys]
print([i + newkeyavg for i in keys])
list(keyindex)
data_cleaned_key_cleanup['keyrelativity'] = data_cleaned_key_cleanup['key']
data_cleaned_key_cleanup.head()
keyrelativity_cleanup = {'keyrelativity':{'C':1.042122547952139,
                                          'FG':1.0377555044170703,
                                          'D':1.0263611698026591,
                                          'AB':1.0193397642267332,
                                          'EF':1.0117356938125703,
                                          'E':0.9916366996451041,
                                          'CD':0.991018721786368,
                                          'A':0.9865457391898016,
                                          'G':0.9786297370945624,
                                          'AB':0.9779352476914113,
                                          'F':0.966123042334426,
                                          'DE':0.9578774520478609}}
data_cleaned_key_cleanup.replace(keyrelativity_cleanup, inplace=True) #only run once
data_cleaned_key_cleanup.tail(20)
data_cleaned_key_cleanup['popularity'].value_counts()
#feature engineering: 'relativity' calculated as some weighted index of 
#popularity and key
data_cleaned_key_cleanup['relativity'] = data_cleaned_key_cleanup['popularity'] * data_cleaned_key_cleanup['keyrelativity']
data_cleaned_key_cleanup.tail(20)
data_cleaned_key_cleanup.dtypes
#change objects to floats
data_cleaned_key_cleanup['keyrelativity']=pd.to_numeric(data_cleaned_key_cleanup['keyrelativity'],errors='coerce')
data_cleaned_key_cleanup.dtypes
data_cleaned_key_cleanup['relativity']=pd.to_numeric(data_cleaned_key_cleanup['relativity'],errors='coerce')
fig = px.treemap(data_cleaned_key_cleanup, path=['year','key','mode'], 
                 values='popularity', color='key', 
                 color_discrete_sequence=px.colors.qualitative.G10,
                 title='Song Popularity by Year, Key and Major/Minor')
fig.show()
fig = px.treemap(data_cleaned_key_cleanup, path=['key','year'], 
                 values='popularity',title='Song Popularity by Key and Year')
fig.show()
fig = px.treemap(data_cleaned_key_cleanup, path=['explicit','year'], 
                 values='popularity',title='Song Popularity by Explicitness and Year')
fig.show()
fig = px.treemap(data_cleaned_key_cleanup, path=['mode','key','year'],
                 values='popularity', color='mode',
                 color_discrete_map={'Major':'gold', 'Minor':'darkblue'},
                 title='Song Popularity by Major/Minor, Key and Year')
fig.show()
fig = px.treemap(data_cleaned_key_cleanup, path=['year','key','mode'], values='relativity',
                 title='Song Relativity by Year, Key and Major/Minor')
fig.show()
type(data_cleaned_key_cleanup)
data_cleaned_key_cleanup.describe()
# according to k means the most predominant features are danceability, energy 
# and liveness
popularity_bins = pd.cut(data_cleaned_key_cleanup['popularity'], bins = [-1,12,33,48,101], labels = ['1','2','3','4'])
danceability_bins = pd.cut(data_cleaned_key_cleanup['danceability'], bins = [-1.0000,0.4170,0.5480,0.6670,1.0010], labels = ['1','2','3','4'])
energy_bins = pd.cut(data_cleaned_key_cleanup['energy'], bins = [-1.0000,0.2630,0.4810,0.7100,1.0010], labels = ['1','2','3','4'])
liveness_bins = pd.cut(data_cleaned_key_cleanup['liveness'], bins = [-1.0000,0.0984,0.1350,0.2630,1.0001], labels = ['1','2','3','4'])
data_cleaned_key_cleanup['pop_b'] = popularity_bins
data_cleaned_key_cleanup['danceability_b'] = danceability_bins
data_cleaned_key_cleanup['energy_b'] = energy_bins
data_cleaned_key_cleanup['liveness_b'] = liveness_bins

data_cleaned_key_cleanup.head()
data_cleaned_key_cleanup.dtypes
data_cleaned_key_cleanup['pop_b']=pd.to_numeric(data_cleaned_key_cleanup['pop_b'],errors='coerce')
data_cleaned_key_cleanup['danceability_b']=pd.to_numeric(data_cleaned_key_cleanup['danceability_b'],errors='coerce')
data_cleaned_key_cleanup['energy_b']=pd.to_numeric(data_cleaned_key_cleanup['energy_b'],errors='coerce')
data_cleaned_key_cleanup['liveness_b']=pd.to_numeric(data_cleaned_key_cleanup['liveness_b'],errors='coerce')
data_cleaned_key_cleanup.dtypes
data_cleaned_key_cleanup['cat_relativity'] = (data_cleaned_key_cleanup['pop_b']
                                              + data_cleaned_key_cleanup['danceability_b']
                                              + data_cleaned_key_cleanup['energy_b']
                                              + data_cleaned_key_cleanup['liveness_b']
                                              + data_cleaned_key_cleanup['relativity'])/5

data_cleaned_key_cleanup.head()
# python plots = data_cleaned_key_cleanup['relativity']
# tfidf = np.array(data_cleaned_key_cleanup['relativity'])
# tfidf_matrix = tfidf.fit_transform(plots)
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
pip install category_encoders
from sklearn.pipeline import make_pipeline
from sklearn.ensemble import RandomForestClassifier
import category_encoders as ce
from sklearn.impute import SimpleImputer
from sklearn.metrics import accuracy_score

processor = make_pipeline(
    ce.OrdinalEncoder(), 
    SimpleImputer(strategy='median'))

model = RandomForestClassifier(n_estimators=30,criterion='gini',max_depth=16,
                           random_state=42, n_jobs=-1,
                           min_impurity_decrease=0)

X = data_cleaned_key_cleanup.drop('name', axis=1)
X = X.drop('explicit', axis=1)
X = X.drop('release_date', axis=1)
X = X.drop('key', axis=1)
X = X.drop('mode', axis=1)
X['X_key'] = data['key']
X['X_mode'] = data['mode']
X.head()
# create feature matrix

y = data_cleaned_key_cleanup['name']

# split data - train/test split
X_train, X_test, y_train, y_test = train_test_split(X, y, 
                                                    test_size=0.20, 
                                                    random_state=42
                                                    )

print(X_train.shape)
print(y_train.shape)
print(X_test.shape)
print(y_test.shape)
# check that the data is random and encoded
X_train.head()
y.head()
import category_encoders as ce
from sklearn.impute import SimpleImputer
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline

pipeline = make_pipeline(
    ce.OrdinalEncoder(), 
    SimpleImputer(strategy='median'), 
    RandomForestClassifier(n_estimators=100, oob_score=True,random_state=42, 
                           n_jobs=-1))

# Fit on train, score on val
pipeline.fit(X_train, y_train)

y_pred = pipeline.predict(X_test)
sub = pd.DataFrame(y_pred)
sub.head()
# convert to interface with application
from joblib import dump
pipeline = model
dump(pipeline, 'pipeline.joblib', compress=True)<jupyter_output><empty_output><jupyter_text>Additional preproccessing of dataPipeline 1 - (instantiate, fit_transform)*   OneHotEncoder*   OrdinalEncoder*   StandardScalerPipeline 2, 3, 4 - (instantiate, fit, predict)*   KNN*   PCA*   t_SNE  Pipeline 1<jupyter_code># normalize standardize data using pipeline (instantiate, fit, transform)
# onehotender, ordinal encoder, standard scaler in order



# Explore each to reduce dimensionality of data: KNN, PCA and t-SNE (instantiate, fit, predict) (tune after MVPs met)<jupyter_output><empty_output><jupyter_text>Pipeline 2<jupyter_code><jupyter_output><empty_output><jupyter_text>Pipeline 3<jupyter_code><jupyter_output><empty_output><jupyter_text>Pipeline 4<jupyter_code><jupyter_output><empty_output><jupyter_text>Transform data to numpy array to feed into NN<jupyter_code># transform data to np array<jupyter_output><empty_output><jupyter_text>Neural Network model Pipeline 1 - (instantiate, compile, fit)*   RNN*   LSTM*   RNN (deep and wide)<jupyter_code><jupyter_output><empty_output><jupyter_text>Evaluate model<jupyter_code><jupyter_output><empty_output><jupyter_text>Vizualize results<jupyter_code><jupyter_output><empty_output><jupyter_text>Conclusions<jupyter_code><jupyter_output><empty_output><jupyter_text>Export to application<jupyter_code># convert to json<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11785407725321888, 'fraction_numerical': 0.042317596566523605, 'mean_word_length': 4.449485500467727, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 51, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '284798', 'license': 'MIT', 'n_tokens_mistral': 4172, 'n_tokens_neox': 3838, 'n_words': 682, 'path': 'Shannon_Li_Spotify_recommender_NN_explore_RJProctor.ipynb', 'repo_name': 'Build-Week-Track-Team-7/explore', 'size': '11674'}