<jupyter_start><jupyter_text>Data Cleaning<jupyter_code>import pandas as pd 
import numpy as np 

# preprocessing for data normalization
from sklearn import preprocessing
# importing sample outputs from predicto() in NN.ipynb

samples = ['6ScgNyiMGRJcuQl6fHE32t',
           '4hpQCCUn1D2KJ0hRIvviNz',
           '1oXRum87ShmIRW8GgETPjd',
           '12pBDYcRz2KJXTnhFste8v',
           '7cGpisHH8TCwcPI1Pxd0IM',
           '3Kb4dMQn8cAXthan2osI0l',
           '2ggqfj97qyiORmXoVFzP5j',
           '2ggqfj97qyiORmXoVFzP5j',
           '3Mt3L75pk83KGc0c4VJzLM',
           '3oDk8PFjkiqwEn1m03pnkm',
           '27fUxjCxoOG7u2kxKAjCJA',
           '63L3A0z2A5DRix83DnHCDX',
           '7hlljw8YiOutMUrFekNIA0',
           '3uZIfWx5ridUBQevmgHDUt']
# path of our song dataset
PATH = '../data/SpotifyFeatures.csv'

df = pd.read_csv(PATH)
df.head()
# wanna keep Danceability, Instrumentalness, Loudness, Speechiness, Valence
# drop all other columns (maybe preserve artist name/track name etc.)

cols = [
    'Unnamed: 0', 'genre', 'popularity', 'duration_ms',
    'key', 'mode', 'tempo', 'time_signature', 'acousticness',
    'energy', 'liveness'
]

df = df.drop(cols, axis=1)
df.head()<jupyter_output><empty_output><jupyter_text>Data Normalization<jupyter_code># our aim here is to make all the data normalized (between 0 and 1)

# let's make a dataframe with just the data to be normalized
slice_df = df.loc[:, 'danceability':'valence']
slice_df.head()
import pandas as pd
from sklearn import preprocessing

x = slice_df.values #returns a numpy array
min_max_scaler = preprocessing.MinMaxScaler()
x_scaled = min_max_scaler.fit_transform(x)
norm_df = pd.DataFrame(x_scaled)
norm_df.head()
# columns got renamed, let's fix that
# Danceability, Instrumentalness, Loudness, Speechiness, Valence
norm_df.columns = ['danceability', 'instrumentalness', 'loudness', 'speechiness', 'valence']
norm_df.head()
# speechiness looks really low, let's see if it's actually between 0 and 1

norm_df['speechiness'].describe() 

# guess so... but it's definitely skewed. oh well.
df = df.loc[:, 'artist_name':'track_id']
df = df.join(norm_df)
df.head()<jupyter_output><empty_output><jupyter_text>Radar Plot Experimentation<jupyter_code>import plotly.graph_objects as go
import numpy as np

def make_graph(track_id_input):

    aa = np.array([track_id_input['danceability']]) # danceability
    bb = np.array([0.000000]) # instrumentalness
    cc = np.array([0.900856]) # loudness
    dd = np.array([0.814000]) # valence
    ee = np.array([0.356292]) # danceability again (to "close" graph)

    categories = ['danceability', 'instrumentalness', 'loudness', 'valence']

    fig = go.Figure()

    fig.add_trace(go.Scatterpolar(
          r=[aa[0], bb[0], cc[0], dd[0], ee[0]],
          theta=categories,
          fill='toself'
    ))

    fig.update_layout(
      polar=dict(
        radialaxis=dict(
          visible=True,
          range=[0, 1]
        )),
          showlegend=False
    )

    fig.show()<jupyter_output><empty_output><jupyter_text>Audio Features:<jupyter_code># Let's start with a spread of each feature over the entire dataset
# We're going to consider this a spread of all music as whole? idk

df.head()
import seaborn as sns
import matplotlib.pyplot as plt 

sns.distplot(df['danceability'])
# sns.distplot(df['instrumentalness'])
sns.distplot(df['loudness'])
# sns.distplot(df['speechiness'])
# sns.distplot(df['valence'])
conn = sqlite3.connect('spotitry_songs.db')
    curs = conn.cursor()
    sql = "INSERT INTO spotitry_songs (id, name, energy, liveness, danceability, instrumentalness, loudness, speechiness, valence, tempo) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
    val = (song_id, name, energy, liveness, danceability, instrumentalness, loudness, speechiness, valence, tempo)
    curs.execute(sql % val)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12551867219917012, 'fraction_numerical': 0.027230290456431536, 'mean_word_length': 4.191117092866756, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '630278', 'license': 'MIT', 'n_tokens_mistral': 1507, 'n_tokens_neox': 1408, 'n_words': 316, 'path': 'visualizations/visualizations.ipynb', 'repo_name': 'spotify-song-suggester1/machine_learning', 'size': '3856'}