<jupyter_start><jupyter_text>Spotify Song Similarity SearchThis notebook takes in a dataset of Spotify songs with several fetures for each song.  We train a KDTree on the dataset, then query the tree for each song's 10 nearest neighbors.  We export the results as a CSV file that can be added (along with the original dataset) to a database for the website.<jupyter_code>import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.neighbors import KDTree
# Import the Spotify dataset as a pandas dataframe
df = pd.read_csv('dataset/SpotifyAudioFeaturesNov2018.csv')
df.head()
## isolating values with a high level of 'speechiness' as they are generally not music but other types of recordings.
## converting unwanted values to null then removing those rows
df['speechiness'].where(df['speechiness'] < .66, inplace=True)
df = df.dropna()
# Create a dataframe the numerical columns that similarity searches will be based on.
df_features = df.drop(columns=['artist_name', 'track_id', 'track_name'])

# Create a numpy array where the numerical features are scaled to a mean of 0 and a standard deviation of 1.
scaler = StandardScaler()
X = scaler.fit_transform(df_features)
# Create the tree
tree = KDTree(X)
%%time
# Query the tree for the 11 nearest neighbors of each song (the first and closest neighbor is itself)
dist, ind = tree.query(X, k=11)
# An overview of the 10 closest neighbors as they become more distant from the x variable.
dist_df = pd.DataFrame(dist)
dist_df.describe()
# Calculate scores from distances
scores = (1 - ((dist - dist.min()) / (dist.max() - dist.min()))) * 100
# Create a list of column headings for the scores
score_columns = ['Searched_Song', 'Song_1', 'Song_2', 'Song_3','Song_4','Song_5','Song_6','Song_7','Song_8','Song_9','Song_10']

# Create a new dataframe of dist scores with column headings
dist_score = pd.DataFrame(scores.tolist(), columns=score_columns)
# Limit decimal place output to 1
dist_score = dist_score.round(decimals=1)
# Review dist_score shape and sample of rows
print(dist_score.shape, '\n')
dist_score.head()
# Replace Searched Song score with the track_id
dist_score['Searched_Song'] = df.track_id
# Check that values have been replaced
dist_score.head()
# ind is an array of the indices of all the nearest neighbors, as found in X and df.
ind[:10]
# ids_only is an array of all the IDs, with the same shape as the original dataframe, 
# so that the indices in ind correspond to the right entries
ids_only = np.array(df.track_id)
ids_only
def process(ids_only, ind): 
    
    similars = []
    
    for row in ind:
        ids = [ids_only[x] for x in row]
        similars.append(ids)
        
    return similars
%%time
similars = process(ids_only, ind)
# The first item in each row is a song in the original database.  
# The next 10 rows are its closest neighbors in the KDTree.
similars[0]
# Turn this list into a dataframe for export
cols = ['id', 'sim1', 'sim2', 'sim3', 'sim4', 'sim5', 'sim6', 'sim7', 'sim8', 'sim9', 'sim10']
df2 = pd.DataFrame(similars, columns=cols)
# Remove duplicates
df3 = df2.drop_duplicates(subset=['id'], keep='first')
# Export to CSV
df3.to_csv('dataset/similars.csv')
# Export distance scores to CSV
dist_score.to_csv('dataset/dist_scores.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09703748488512697, 'fraction_numerical': 0.016021765417170496, 'mean_word_length': 4.835978835978836, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '555275', 'license': 'MIT', 'n_tokens_mistral': 1078, 'n_tokens_neox': 1003, 'n_words': 411, 'path': 'similarity_search.ipynb', 'repo_name': 'bw-spotify/data-science', 'size': '3308'}