<jupyter_start><jupyter_text>ML Models on Timbre Data  Loading more or less clean data<jupyter_code>import numpy as np
import pandas as pd 
import matplotlib.pyplot as plt
%matplotlib inline
df_timbre = pd.read_csv('TimbreAll_withFeatures.csv')
df_timbre.head()
df_timbre.columns
# Drop columns
df_timbre = df_timbre.drop(['Unnamed: 0', 'Unnamed: 0.1', 'timbre_all_1', 'timbre_all_2',
       'timbre_all_3', 'timbre_all_4', 'timbre_all_5', 'timbre_all_6',
       'timbre_all_7', 'timbre_all_8', 'timbre_all_9', 'timbre_all_10',
       'timbre_all_11', 'timbre_all_12'], axis=1)
df_timbre.head()
df_global = pd.read_csv('cleanGlobalPrediction1.csv',encoding='latin-1')
df_global.head()
# Define dataframe
df_raw = pd.concat([df_global, df_timbre] , axis = 1)
df_raw.head()
df_raw.columns
df = df_raw.drop(['Unnamed: 0','analysis_url', 'track_href', 'uri'], axis =1)
print(df.shape)
df.head()
# Devide popularity into quantile bins
df['popularity'] = pd.qcut(df['popularity'], 3, labels= False)
df['popularity']
# Drop middle quantile to create gap
df_reduced = df[df.popularity!=1]
print('Reduced data set to:')
print(df.shape[0])
features = list(df.columns)
features.remove('global')
features.remove('global_100')
features.remove('global_top')
features.remove('popularity')

features.remove('song')
features.remove('artist')
print('Those are our {} features'.format(len(features)))
print(features)<jupyter_output>Those are our 110 features
['acousticness', 'danceability', 'duration_ms', 'energy', 'instrumentalness', 'key', 'liveness', 'loudness', 'mode', 'speechiness', 'tempo', 'valence', 'International', 'Unknown', 'electronic', 'folk', 'hip hop', 'house', 'indie', 'latino', 'metal', 'pop', 'punk', 'r&b', 'rap', 'rock', 'mean_timbre1', 'median_timbre1', 'std_timbre1', 'min_timbre1', 'max_timbre1', 'range_timbre1', '80Percentile_timbre1', 'mean_timbre2', 'median_timbre2', 'std_timbre2', 'min_timbre2', 'max_timbre2', 'range_timbre2', '80Percentile_timbre2', 'mean_timbre3', 'median_timbre3', 'std_timbre3', 'min_timbre3', 'max_timbre3', 'range_timbre3', '80Percentile_timbre3', 'mean_timbre4', 'median_timbre4', 'std_timbre4', 'min_timbre4', 'max_timbre4', 'range_timbre4', '80Percentile_timbre4', 'mean_timbre5', 'median_timbre5', 'std_timbre5', 'min_timbre5', 'max_timbre5', 'range_timbre5', '80Percentile_timbre5', 'mean_timbre6', 'median_timbre6', 'std_timbre6', 'min_timbre6', 'max_timbre6', 'range_[...]<jupyter_text>Machine Learning  Define Input/Output<jupyter_code># Clean dataframe
df_reduced.head()
# Input
X= df_reduced[features]
X.head()
print(X.shape)
# Normalize Input
from sklearn.preprocessing import normalize
X_norm = normalize(X)
# Output
Y = df_reduced['popularity']
print(Y.shape)
Y.head()
# Train-Test Split
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(X,Y, test_size=0.15, random_state=100)
# Confusion Matrix
from sklearn.metrics import confusion_matrix
def printConfusionMatrix (y_true, y_pred):
    cf=pd.DataFrame(confusion_matrix(y_true, y_pred))
    print(cf)<jupyter_output><empty_output><jupyter_text>1. Logistic Regression<jupyter_code>from sklearn.linear_model import LogisticRegression
print('Logistic Regression on popularity: ')
logRegModel = LogisticRegression()
logRegModel.fit(x_train, y_train)
# Accuracy
print('Train Accuracy:')
print(logRegModel.score(x_train, y_train))
print('Test Accuracy:')
print(logRegModel.score(x_test, y_test))
print('Confusion Matrix')
printConfusionMatrix(y_test, logRegModel.predict(x_test))<jupyter_output>Logistic Regression on popularity: 
Train Accuracy:
0.76
Test Accuracy:
0.6296296296296297
Confusion Matrix
    0   1
0  21   7
1  13  13<jupyter_text>2. Random Forest<jupyter_code>from sklearn.ensemble import RandomForestClassifier
random_forest = RandomForestClassifier(n_estimators=190)
random_forest.fit(x_train, y_train)
acc_rf1 = random_forest.score(x_train, y_train)
acc_rf2 = random_forest.score(x_test, y_test)
print("Random Forest: ")
print("Training: ", acc_rf1)
print("Test: ", acc_rf2)
print('Confusion Matrix')
printConfusionMatrix(y_test, random_forest.predict(x_test))
values = random_forest.feature_importances_
dict_featureImportance = dict(zip(features,values))
dict_featureImportance
import operator
sorted_dict = sorted(dict_featureImportance.items(), key=operator.itemgetter(1), reverse = True)
sorted_dict<jupyter_output><empty_output><jupyter_text>Predict global_100<jupyter_code>Y = df_reduced['global_100']
Y.head()
Y.value_counts()
# Train-Test Split
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(X,Y, test_size=0.15, random_state=100)
from sklearn.linear_model import LogisticRegression
print('Logistic Regression on global_100: ')
logRegModel = LogisticRegression()
logRegModel.fit(x_train, y_train)
# Accuracy
print('Train Accuracy:')
print(logRegModel.score(x_train, y_train))
print('Test Accuracy:')
print(logRegModel.score(x_test, y_test))
print('Confusion Matrix')
printConfusionMatrix(y_test, logRegModel.predict(x_test))<jupyter_output>Logistic Regression on global_100: 
Train Accuracy:
0.6966666666666667
Test Accuracy:
0.6481481481481481
Confusion Matrix
    0   1
0  24   4
1  15  11<jupyter_text>Predict Overall Success Factor<jupyter_code>a = 10
b = 20
c = 100
Y = df.loc[:,'global']*a+df.global_100*b+df.global_top*c+df.popularity
Y.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13658176448874124, 'fraction_numerical': 0.037467700258397935, 'mean_word_length': 8.591150442477876, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '220723', 'license': 'MIT', 'n_tokens_mistral': 2137, 'n_tokens_neox': 1997, 'n_words': 384, 'path': 'ML Models/Timbre Approach/.ipynb_checkpoints/ModelsOnTimbre_firstGlance-checkpoint.ipynb', 'repo_name': 'liams32/Spotify_Analysis', 'size': '5418'}