<jupyter_start><jupyter_text>SpotifinderSong recommender model based on Natural Language Processing.Using TFIDF and Autoencoder or Latent Semantic Indexing. Slimmed down model using autoencoder.<jupyter_code>#libraries imported

import pandas as pd
import re
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.neighbors import NearestNeighbors
import pickle
from wordcloud import WordCloud, STOPWORDS
import matplotlib.pyplot as plt
# NLP model used this 18k songs dataset
# Trim it down by taking out duplicate songs

spotify_songs = pd.read_csv(r'C:\Users\temsy\Documents\GitHub\Spotifinder_BU3\data\spotify_songs.csv')
spotify_songs.shape
#Data cleaning

def clean_data(data):
    #drop nulls in lyrics column
    data = data.dropna()
    
    #Only get songs with lyrics in English
    data = data[data['language']=='en']
    
    #Drop duplicates
    data = data.drop_duplicates(subset = ['track_name', 'track_artist'],
                                             keep = 'first')
    
    #Reduce features
    features = ['track_id', 'track_name', 'track_artist', 'lyrics']
    data = data[features]
    
    #Reset index
    data = data.reset_index()
    
    return data
df = clean_data(spotify_songs)
df.shape
#Export new dataset to csv

df.to_csv('NLP_songs_data.csv', index=False)
df = pd.read_csv('NLP_songs_data.csv')
df.shape
df.head(10)<jupyter_output><empty_output><jupyter_text>Create TF-IDF Model<jupyter_code>#Create a function that gathers all the song lyrics into a list of strings

def gather_data(songs):
    data =[]
    for song in songs:
        data.append(df['lyrics'][df['track_id']==song])
    
    #Convert Series into a list of strings
    new_data = []

    for song in data:
        str_song = pd.Series(song).item()
        new_data.append(str_song)
        
    return new_data
#Gather all the songs in the dataframe

songs = df['track_id']

data = gather_data(songs)
type(data)
len(data)

#Now this is a corpus
# Apply TD-IDF to the corpus 
# Turn this on once to create the dtm, 
# it will be saved in the next cell 

# Instantiate 
tfidf = TfidfVectorizer(stop_words='english', 
                        ngram_range=(1,2), min_df=0.03, 
                        max_df=0.25) 

# Create a vocabulary and get word counts per document 
dtm = tfidf.fit_transform(data) 

# Get feature names to use as dataframe column headers 
dtm = pd.DataFrame(dtm.todense(), columns=tfidf.get_feature_names()) 
print(dtm.shape) 
dtm.head()
# Serialize the dtm 
#Turn this on once to save the dtm 

dtm_filename = 'nlp_dtm.pkl' 
pickle.dump(dtm, open(dtm_filename, 'wb'))
#Load and test the dtm

dtm_filename = 'nlp_dtm.pkl'

dtm = pickle.load(open(dtm_filename, 'rb'))
# Apply K Nearest Neighbors 
# Fit on DTM 

nn = NearestNeighbors(n_neighbors=6, algorithm='kd_tree') 
nn.fit(dtm)
#Serialize model 
#Turn this on once to save the model 

model_filename = 'nlp_model.pkl' 
pickle.dump(nn, open(model_filename, 'wb'))
#Load and test the model

model_filename = r'C:\Users\temsy\Documents\GitHub\Spotifinder_BU3\models\nlp_model.pkl'

loaded_model = pickle.load(open(model_filename, 'rb'))
# result = loaded_model.kneighbors([doc])
# print(result)
#function to get 5 nearest songs

def select_nearest_songs(artist, song):
    
    #translate artist, song into doc dtm.iloc[x].values
    artist_songs = df.loc[df['track_artist']==artist]
    selected_song = artist_songs.loc[artist_songs['track_name']==song]
    x = selected_song.index
    x = x[0]
    x = x.item()

    doc = dtm.loc[x].values
    result = loaded_model.kneighbors([doc], n_neighbors=6)
    
    rec_songs = {"artist":[], "song":[]};
    
    for i in range(5):
        song = result[1][0][1+i]

        #translate the loc into an artist and song title
        artist = df.loc[song]['track_artist']
        song = df.loc[song]['track_name']
        
        rec_songs['artist'].append(artist)
        rec_songs['song'].append(song)
    
    return rec_songs
select_nearest_songs('The Weeknd', 'Heartless')
select_nearest_songs("CeeLo Green", "Baby It's Cold Outside (feat. Christina Aguilera)")
select_nearest_songs('Babyface', 'Tender Lover')<jupyter_output><empty_output><jupyter_text>Compare Lyrics<jupyter_code>#Fetch lyrics function

def get_lyrics(artist, song):
  songs_by_artist = df[['track_id','track_name']][df['track_artist'] == artist]
  song_id = songs_by_artist[songs_by_artist['track_name'] == song]
  song_id = song_id[:1] #This selects the first if there are more than one
  song_id = song_id['track_id']
  song_id = pd.Series(song_id).item()
  lyrics = df['lyrics'][df['track_id'] == song_id]
  lyrics = pd.Series(lyrics).item()
  return lyrics
#Print lyrics from both songs to compare

def compare_lyrics(artist, song):
    song_recs = select_nearest_songs(artist, song)
    
    rec_artist = song_recs['artist'][0]
    rec_song = song_recs['song'][0]
    
    lyrics1 = get_lyrics(artist, song)
    lyrics2 = get_lyrics(rec_artist, rec_song)
    
    return lyrics1, lyrics2<jupyter_output><empty_output><jupyter_text>Visualization 1: Word Cloud<jupyter_code>stopwords = set(STOPWORDS)

def show_wordcloud(data, title = None):
    wordcloud = WordCloud(
        background_color='white',
        stopwords=stopwords,
        max_words=50,
        max_font_size=40,
        scale=3,
        random_state=37).generate(str(data))
    
    fig = plt.figure(1, figsize=(12,12))
    plt.axis('off')
    if title:
        fig.suptitle(title, fontsize=20)
        fig.subplots_adjust(top=2.3)
        
    plt.imshow(wordcloud)
    plt.show()
lyrics1, lyrics2 = compare_lyrics('The Weeknd', 'Heartless')

show_wordcloud(lyrics1)
show_wordcloud(lyrics2)
lyrics1, lyrics2 = compare_lyrics('CeeLo Green', "Baby It's Cold Outside (feat. <NAME>)")

show_wordcloud(lyrics1)
show_wordcloud(lyrics2)
lyrics1, lyrics2 = compare_lyrics('Babyface', 'Tender Lover')

show_wordcloud(lyrics1)
show_wordcloud(lyrics2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11372417268604065, 'fraction_numerical': 0.009742986729380145, 'mean_word_length': 3.9824267782426777, 'pattern_counts': {'":': 2, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '86108', 'license': 'MIT', 'n_tokens_mistral': 2184, 'n_tokens_neox': 2116, 'n_words': 481, 'path': 'notebooks/NLP2_model.ipynb', 'repo_name': 'TemsyChen/Spotifinder', 'size': '5965'}