Add application file

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.csv filter=lfs diff=lfs merge=lfs -text

Data/new_tracks.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:691f1905395f220b953dc4e259a7bfbba8a17e4e75923ca955fe9d3561ff3626
+size 22648

Data/streamlit.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6a69e37d071948e19528b8b37eab0ac2c96dbf7ff846ddc630a5539dcded952
+size 187020124

README.md

@@ -5,7 +5,7 @@ colorFrom: red
 colorTo: gray
 sdk: streamlit
 sdk_version: 1.10.0
-app_file: app.py
+app_file: main.py
 pinned: false
 ---
 

Spotify/Spotify.yaml

@@ -0,0 +1,2 @@
+Client_id : 64049e33e24744f7a0a30e945e37be6c
+client_secret : c9b30ec81f8d40ac81d5dd6aeec45ee4

Spotify/Spotify1.yaml

@@ -0,0 +1,2 @@
+Client_id : 28a753df0eb7494e8c2d1b212f20c590
+client_secret : 9685313e8178455e9d4f05893ffe26e8

Spotify/Spotify2.yaml

@@ -0,0 +1,2 @@
+Client_id : e5d5709d13204cb9bdc3cae44bc6f39b
+client_secret : f0f31bee6b584b7083fec26717707c69

Spotify/Spotify3.yaml

@@ -0,0 +1,2 @@
+Client_id : cc37f3ceb5ae47949933f41be465fbc8
+client_secret : 1f8095e1e71f42848f00187132cb6e95

Spotify/Spotify4.yaml

@@ -0,0 +1,2 @@
+Client_id : aa397cd7e24e4c56a199f433692e48f3
+client_secret : d85b2dfc58784376b5b33f94393b5d5e

Spotify/Spotify5.yaml

@@ -0,0 +1,2 @@
+Client_id : bfa5e7b3d8424aa28ef274c85e104c44
+client_secret : 49e7e21f73ca4665be2294cfe10e66d3

Spotify/Spotify6.yaml

@@ -0,0 +1,2 @@
+Client_id : 65719a2ea2104545aa4b03a3ef622a6d
+client_secret : a388bb9c73e34dcdad99814efa21a404

main.py

@@ -0,0 +1,293 @@
+import streamlit as st
+from streamlit_option_menu import option_menu
+import streamlit.components.v1 as components
+import time
+from model import *
+
+
+
+if 'model' not in st.session_state:
+    st.session_state.model = 'Model 1'
+def update_radio2():
+    st.session_state.model=st.session_state.radio2
+if 'genre' not in st.session_state:
+    st.session_state.genre=3
+def update_num_genre():
+    st.session_state.genre=st.session_state.num_genre
+if 'artist' not in st.session_state:
+    st.session_state.artist=5
+def update_same_art():
+    st.session_state.artist=st.session_state.same_art
+if 'model2' not in st.session_state:
+    st.session_state.model2= 'Spotify model'
+def update_radio1():
+    st.session_state.model2 =st.session_state.radio1
+
+if 'Region' not in st.session_state:
+    st.session_state.rg="US"
+def update_Region():
+    st.session_state.rg=st.session_state.Region
+if 'radio' not in st.session_state:
+    st.session_state.feature="Playlist"
+def update_radio0():
+    st.session_state.feature=st.session_state.radio
+
+if 'p_url' not in st.session_state:
+    st.session_state.p_url = 'Example: https://open.spotify.com/playlist/37i9dQZF1DX8FwnYE6PRvL?si=06ff6b38d4124af0'
+def update_playlist_url():
+    st.session_state.p_url = st.session_state.playlist_url
+
+if 's_url' not in st.session_state:
+    st.session_state.s_url = 'Example: https://open.spotify.com/track/5CQ30WqJwcep0pYcV4AMNc?si=ed4b04f153a24531'
+def update_song_url():
+    st.session_state.s_url = st.session_state.song_url
+
+if 'a_url' not in st.session_state:
+    st.session_state.a_url = 'Example: https://open.spotify.com/artist/3RNrq3jvMZxD9ZyoOZbQOD?si=UNAsX20kRpG89bxOO8o7ew'
+def update_artist_url():
+    st.session_state.a_url = st.session_state.artist_url
+
+
+def play_recomm():
+    if 'rs' in st.session_state:
+        del st.session_state.rs,st.session_state.err
+    try:
+        if len(pd.read_csv('Data/new_tracks.csv')) >= 200:
+            with st.spinner('Updating the dataset...'):
+                x=update_dataset()
+                st.success('{} New tracks were added to the dataset.'.format(x))
+    except:
+        st.error("The dataset update failed. ")
+    with st.spinner('Getting Recommendations...'):
+        res,err = playlist_model(st.session_state.p_url,st.session_state.model,st.session_state.genre,st.session_state.artist)
+        st.session_state.rs=res
+        st.session_state.err=err
+    if len(st.session_state.rs)>=1:
+        if st.session_state.model == 'Model 1' or st.session_state.model == 'Model 2':
+            st.success('Go to the Result page to view the top {} recommendations'.format(len(st.session_state.rs)))
+        else:
+            st.success('Go to the Result page to view the  Spotify recommendations')
+    else:
+        st.error('Model failed. Check the log for more information.')   
+
+def art_recomm():
+    if 'rs' in st.session_state:
+        del st.session_state.rs,st.session_state.err
+    with st.spinner('Getting Recommendations...'):
+        res,err = top_tracks(st.session_state.a_url,st.session_state.rg)
+        st.session_state.rs=res
+        st.session_state.err=err
+    if len(st.session_state.rs)>=1:
+        st.success("Go to the Result page to view the Artist's top tracks")
+    else:
+        st.error('Model failed. Check the log for more information.')
+
+def song_recomm():
+    if 'rs' in st.session_state:
+        del st.session_state.rs,st.session_state.err
+    with st.spinner('Getting Recommendations...'):
+        res,err = song_model(st.session_state.s_url,st.session_state.model,st.session_state.genre,st.session_state.artist)
+        st.session_state.rs=res
+        st.session_state.err=err
+    if len(st.session_state.rs)>=1:
+        if st.session_state.model == 'Model 1' or st.session_state.model == 'Model 2':
+            st.success('Go to the Result page to view the top {} recommendations'.format(len(st.session_state.rs)))
+        else:
+            st.success('Go to the Result page to view the  Spotify recommendations')
+    else:
+        st.error('Model failed. Check the log for more information.')
+
+def playlist_page():
+    st.subheader("User Playlist")
+    st.markdown('---')
+    playlist_uri = (st.session_state.playlist_url).split('/')[-1].split('?')[0]
+    uri_link = 'https://open.spotify.com/embed/playlist/' + playlist_uri
+    components.iframe(uri_link, height=300)
+    return
+
+def song_page():
+    st.subheader("User Song")
+    st.markdown('---')
+    song_uri = (st.session_state.song_url).split('/')[-1].split('?')[0]
+    uri_link = 'https://open.spotify.com/embed/track/' + song_uri
+    components.iframe(uri_link, height=100)
+
+def artist_page():
+    st.subheader("User Artist")
+    st.markdown('---')
+    artist_uri = (st.session_state.artist_url).split('/')[-1].split('?')[0]
+    uri_link = 'https://open.spotify.com/embed/artist/' + artist_uri
+    components.iframe(uri_link, height=80)
+
+
+def spr_sidebar():
+    menu=option_menu(
+        menu_title=None,
+        options=['Home','Result','About','Log'],
+        icons=['house','book','info-square','terminal'],
+        menu_icon='cast',
+        default_index=0,
+        orientation='horizontal'
+    )
+    if menu=='Home':
+        st.session_state.app_mode = 'Home'
+    elif menu=='Result':
+        st.session_state.app_mode = 'Result'
+    elif menu=='About':
+        st.session_state.app_mode = 'About'
+    elif menu=='Log':
+        st.session_state.app_mode = 'Log'
+    
+def home_page():
+    st.session_state.radio=st.session_state.feature
+    st.session_state.radio2=st.session_state.model
+    st.session_state.num_genre=st.session_state.genre
+    st.session_state.same_art=st.session_state.artist
+    st.session_state.Region=st.session_state.rg
+
+    
+    st.title('Spotify Recommendation System')
+    col,col2,col3=st.columns([2,2,3])
+    radio=col.radio("Feature",options=("Playlist","Song","Artist Top Tracks"),key='radio',on_change=update_radio0)
+    if radio =="Artist Top Tracks":
+        radio1=col2.radio("Model",options=["Spotify model"],key='radio1',on_change=update_radio1)
+        Region=col3.selectbox("Please Choose Region",index=58,key='Region',on_change=update_Region,options=('AD', 'AR', 'AU', 'AT', 'BE', 'BO', 'BR', 'BG', 'CA', 'CL', 'CO', 'CR', 'CY', 'CZ', 'DK', 'DO', 'EC', 'SV', 'EE', 'FI', 'FR', 'DE', 'GR', 'GT', 'HN', 'HK', 'HU', 'IS', 'ID', 'IE', 'IT', 'JP', 'LV', 'LI', 'LT', 'LU', 'MY', 'MT', 'MX', 'MC', 'NL', 'NZ', 'NI', 'NO', 'PA', 'PY', 'PE', 'PH', 'PL', 'PT', 'SG', 'ES', 'SK', 'SE', 'CH', 'TW', 'TR', 'GB', 'US', 'UY'))
+    elif radio =="Playlist" or radio =="Song" :
+        radio2=col2.radio("Model",options=("Model 1","Model 2","Spotify Model"),key='radio2',on_change=update_radio2)
+        if st.session_state.radio2=="Model 1" or st.session_state.radio2=="Model 2":
+            num_genre=col3.selectbox("choose a number of genres to focus on",options=(1,2,3,4,5,6,7),index=2,key='num_genre',on_change=update_num_genre)
+            same_art=col3.selectbox("How many recommendations by the same artist",options=(1,2,3,4,5,7,10,15),index=3,key='same_art',on_change=update_same_art)
+
+
+    st.markdown("<br>", unsafe_allow_html=True)
+    
+    if radio == "Playlist" :
+        st.session_state.playlist_url = st.session_state.p_url
+        Url = st.text_input(label="Playlist Url",key='playlist_url',on_change=update_playlist_url)
+        playlist_page()
+        state =st.button('Get Recommendations')
+        with st.expander("Here's how to find any Playlist URL in Spotify"):
+            st.write(""" 
+                - Search for Playlist on the Spotify app
+                - Right Click on the Playlist you like
+                - Click "Share"
+                - Choose "Copy link to playlist"
+            """)
+            st.markdown("<br>", unsafe_allow_html=True)
+            st.image('spotify_get_playlist_url.png')
+        if state:
+            play_recomm()
+    elif radio == "Song" :
+        st.session_state.song_url = st.session_state.s_url
+        Url = st.text_input(label="Song Url",key='song_url',on_change=update_song_url)
+        song_page()
+        state =st.button('Get Recommendations')
+        with st.expander("Here's how to find any Song URL in Spotify"):
+            st.write(""" 
+                - Search for Song on the Spotify app
+                - Right Click on the Song you like
+                - Click "Share"
+                - Choose "Copy link to Song"
+            """)
+            st.markdown("<br>", unsafe_allow_html=True)
+            st.image('spotify_get_song_url.png')
+        if state:
+            song_recomm()
+    elif radio == "Artist Top Tracks" :
+        st.session_state.artist_url = st.session_state.a_url
+        Url = st.text_input(label="Artist Url",key='artist_url',on_change=update_artist_url)
+        artist_page()
+        state =st.button('Get Recommendations')
+        with st.expander("Here's how to find any Artist URL in Spotify"):
+            st.write(""" 
+                - Search for Artist on the Spotify app
+                - Right Click on the Artist you like
+                - Click "Share"
+                - Choose "Copy link to Artist"
+            """)
+            st.markdown("<br>", unsafe_allow_html=True)
+            st.image('spotify_get_artist_url.png')
+        if state:
+            art_recomm()
+    
+def result_page():
+    if 'rs' not in st.session_state:
+        st.error('Please select a model on the Home page and run Get Recommendations')
+    else:
+        st.success('Top {} recommendations'.format(len(st.session_state.rs)))
+        i=0
+        for uri in st.session_state.rs:
+         uri_link = "https://open.spotify.com/embed/track/" + uri + "?utm_source=generator&theme=0"
+         components.iframe(uri_link, height=80)
+         i+=1
+         if i%5==0:
+            time.sleep(1)
+def Log_page():
+    log=st.checkbox('Display Output', True, key='display_output')
+    if log == True:
+     if 'err' in st.session_state:
+        st.write(st.session_state.err)
+    with open('Data/new_tracks.csv') as f:
+        st.download_button('Download Dataset', f,file_name='new_tracks.csv')
+def About_page():
+    st.header('Development')
+    """
+    Check out the [repository](https://github.com/abdelrhmanelruby/Spotify-Recommendation-System) for the source code and approaches, and don't hesitate to contact me if you have any questions. I'm excited to read your review.
+    [Github](https://github.com/abdelrhmanelruby)  [Linkedin](https://www.linkedin.com/in/abdelrhmanelruby/) Email : [email protected]
+    """
+    st.subheader('Spotify Million Playlist Dataset')
+    """
+    For this project, I'm using the Million Playlist Dataset, which, as its name implies, consists of one million playlists.
+    contains a number of songs, and some metadata is included as well, such as the name of the playlist, duration, number of songs, number of artists, etc.
+    """
+
+    """
+    It is created by sampling playlists from the billions of playlists that Spotify users have created over the years. 
+    Playlists that meet the following criteria were selected at random:
+    - Created by a user that resides in the United States and is at least 13 years old
+    - Was a public playlist at the time the MPD was generated
+    - Contains at least 5 tracks
+    - Contains no more than 250 tracks
+    - Contains at least 3 unique artists
+    - Contains at least 2 unique albums
+    - Has no local tracks (local tracks are non-Spotify tracks that a user has on their local device
+    - Has at least one follower (not including the creator
+    - Was created after January 1, 2010 and before December 1, 2017
+    - Does not have an offensive title
+    - Does not have an adult-oriented title if the playlist was created by a user under 18 years of age
+
+    Information about the Dataset [here](https://www.aicrowd.com/challenges/spotify-million-playlist-dataset-challenge)
+    """
+    st.subheader('Audio Features Explanation')
+    """
+    | Variable | Description |
+    | :----: | :---: |
+    | Acousticness | A confidence measure from 0.0 to 1.0 of whether the track is acoustic. 1.0 represents high confidence the track is acoustic. |
+    | Danceability | Danceability describes how suitable a track is for dancing based on a combination of musical elements including tempo, rhythm stability, beat strength, and overall regularity. A value of 0.0 is least danceable and 1.0 is most danceable. |
+    | Energy | Energy is a measure from 0.0 to 1.0 and represents a perceptual measure of intensity and activity. Typically, energetic tracks feel fast, loud, and noisy. For example, death metal has high energy, while a Bach prelude scores low on the scale. Perceptual features contributing to this attribute include dynamic range, perceived loudness, timbre, onset rate, and general entropy. |
+    | Instrumentalness | Predicts whether a track contains no vocals. "Ooh" and "aah" sounds are treated as instrumental in this context. Rap or spoken word tracks are clearly "vocal". The closer the instrumentalness value is to 1.0, the greater likelihood the track contains no vocal content. Values above 0.5 are intended to represent instrumental tracks, but confidence is higher as the value approaches 1.0. |
+    | Key | The key the track is in. Integers map to pitches using standard Pitch Class notation. E.g. 0 = C, 1 = C♯/D♭, 2 = D, and so on. If no key was detected, the value is -1. |
+    | Liveness | Detects the presence of an audience in the recording. Higher liveness values represent an increased probability that the track was performed live. A value above 0.8 provides strong likelihood that the track is live. |
+    | Loudness | The overall loudness of a track in decibels (dB). Loudness values are averaged across the entire track and are useful for comparing relative loudness of tracks. Loudness is the quality of a sound that is the primary psychological correlate of physical strength (amplitude). Values typically range between -60 and 0 db. |
+    | Mode | Mode indicates the modality (major or minor) of a track, the type of scale from which its melodic content is derived. Major is represented by 1 and minor is 0. |
+    | Speechiness | Speechiness detects the presence of spoken words in a track. The more exclusively speech-like the recording (e.g. talk show, audio book, poetry), the closer to 1.0 the attribute value. Values above 0.66 describe tracks that are probably made entirely of spoken words. Values between 0.33 and 0.66 describe tracks that may contain both music and speech, either in sections or layered, including such cases as rap music. Values below 0.33 most likely represent music and other non-speech-like tracks. |
+    | Tempo | The overall estimated tempo of a track in beats per minute (BPM). In musical terminology, tempo is the speed or pace of a given piece and derives directly from the average beat duration. |
+    | Time Signature | An estimated time signature. The time signature (meter) is a notational convention to specify how many beats are in each bar (or measure). The time signature ranges from 3 to 7 indicating time signatures of "3/4", to "7/4". |
+    | Valence | A measure from 0.0 to 1.0 describing the musical positiveness conveyed by a track. Tracks with high valence sound more positive (e.g. happy, cheerful, euphoric), while tracks with low valence sound more negative (e.g. sad, depressed, angry). |
+    
+    Information about features: [here](https://developer.spotify.com/documentation/web-api/reference/#/operations/get-audio-features)
+    """
+
+def main():
+    spr_sidebar()        
+    if st.session_state.app_mode == 'Home':
+        home_page()
+    if st.session_state.app_mode == 'Result':
+        result_page()
+    if st.session_state.app_mode == 'About' :
+        About_page()
+    if st.session_state.app_mode == 'Log':
+        Log_page()
+# Run main()
+if __name__ == '__main__':
+    main()

model.py

@@ -0,0 +1,481 @@
+import pandas as pd
+import spotipy
+from spotipy.oauth2 import SpotifyOAuth, SpotifyClientCredentials
+import yaml
+import re
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+from sklearn.preprocessing import MinMaxScaler
+import pickle
+
+def playlist_model(url, model, max_gen=3, same_art=5):
+    log = []
+    Fresult = []
+    try:
+     log.append('Start logging')
+     uri = url.split('/')[-1].split('?')[0]
+     stream = open("Spotify/Spotify.yaml")
+     spotify_details = yaml.safe_load(stream)
+     auth_manager = SpotifyClientCredentials(
+         client_id=spotify_details['Client_id'], client_secret=spotify_details['client_secret'])
+     sp = spotipy.client.Spotify(auth_manager=auth_manager) 
+
+     if model == 'Spotify Model':
+         def get_IDs(user, playlist_id):
+             try:
+                 log.append('start playlist extraction')
+                 track_ids = []
+                 playlist = sp.user_playlist(user, playlist_id)
+                 for item in playlist['tracks']['items']:
+                     track = item['track']
+                     track_ids.append(track['id'])
+                 return track_ids
+             except Exception as e:
+                 log.append('Failed to load the playlist')
+                 log.append(e)
+
+         track_ids = get_IDs('Ruby', uri)
+         track_ids_uni = list(set(track_ids))
+         log.append('Starting Spotify Model')
+         Spotifyresult = pd.DataFrame()
+         for i in range(len(track_ids_uni)-5):
+             if len(Spotifyresult) >= 50:
+                 break
+             try:
+                 ff = sp.recommendations(seed_tracks=list(track_ids_uni[i:i+5]), limit=5)
+             except Exception as e:
+                 log.append(e)
+                 continue
+             for z in range(5):
+                 result = pd.DataFrame([z+(5*i)+1])
+                 result['uri'] = ff['tracks'][z]['id']
+                 Spotifyresult = pd.concat([Spotifyresult, result], axis=0)
+                 Spotifyresult.drop_duplicates(subset=['uri'], inplace=True,keep='first')
+             Fresult = Spotifyresult.uri[:50]
+
+         log.append('Model run successfully')
+         return Fresult, log
+
+     lendf=len(pd.read_csv('Data/streamlit.csv',usecols=['track_uri']))
+     dtypes = {'track_uri': 'object', 'artist_uri': 'object', 'album_uri': 'object', 'danceability': 'float16', 'energy': 'float16', 'key': 'float16',
+               'loudness': 'float16', 'mode': 'float16', 'speechiness': 'float16', 'acousticness': 'float16', 'instrumentalness': 'float16',
+               'liveness': 'float16', 'valence': 'float16', 'tempo': 'float16', 'duration_ms': 'float32', 'time_signature': 'float16',
+               'Track_release_date': 'int8', 'Track_pop': 'int8', 'Artist_pop': 'int8', 'Artist_genres': 'object'}
+     col_name= ['track_uri', 'artist_uri', 'album_uri', 'danceability', 'energy', 'key',
+        'loudness', 'mode', 'speechiness', 'acousticness', 'instrumentalness',
+        'liveness', 'valence', 'tempo', 'duration_ms', 'time_signature',
+        'Track_release_date', 'Track_pop', 'Artist_pop', 'Artist_genres']
+    
+     try:
+         def get_IDs(user, playlist_id):
+             log.append('start playlist extraction')
+             track_ids = []
+             artist_id = []
+             playlist = sp.user_playlist(user, playlist_id)
+             for item in playlist['tracks']['items']:
+                 track = item['track']
+                 track_ids.append(track['id'])
+                 artist = item['track']['artists']
+                 artist_id.append(artist[0]['id'])
+             return track_ids, artist_id
+     except Exception as e:
+         log.append('Failed to load the playlist')
+         log.append(e)
+
+     track_ids, artist_id = get_IDs('Ruby', uri)
+     log.append("Number of Track : {}".format(len(track_ids)))
+
+     artist_id_uni = list(set(artist_id))
+     track_ids_uni = list(set(track_ids))
+     log.append("Number of unique Artists : {}".format(len(artist_id_uni)))
+     log.append("Number of unique Tracks : {}".format(len(track_ids_uni)))
+
+     def extract(track_ids_uni, artist_id_uni):
+         err = []
+         err.append('Start audio features extraction')
+         audio_features = pd.DataFrame()
+         for i in range(0, len(track_ids_uni), 25):
+             try:
+                 track_feature = sp.audio_features(track_ids_uni[i:i+25])
+                 track_df = pd.DataFrame(track_feature)
+                 audio_features = pd.concat([audio_features, track_df], axis=0)
+             except Exception as e:
+                 err.append(e)
+                 continue
+         err.append('Start track features extraction')
+         track_ = pd.DataFrame()
+         for i in range(0, len(track_ids_uni), 25):
+             try:
+                 track_features = sp.tracks(track_ids_uni[i:i+25])
+                 for x in range(25):
+                     track_pop = pd.DataFrame([track_ids_uni[i+x]], columns=['Track_uri'])
+                     track_pop['Track_release_date'] = track_features['tracks'][x]['album']['release_date']
+                     track_pop['Track_pop'] = track_features['tracks'][x]["popularity"]
+                     track_pop['Artist_uri'] = track_features['tracks'][x]['artists'][0]['id']
+                     track_pop['Album_uri'] = track_features['tracks'][x]['album']['id']
+                     track_ = pd.concat([track_, track_pop], axis=0)
+             except Exception as e:
+                 err.append(e)
+                 continue
+         err.append('Start artist features extraction')
+         artist_ = pd.DataFrame()
+         for i in range(0, len(artist_id_uni), 25):
+             try:
+                 artist_features = sp.artists(artist_id_uni[i:i+25])
+                 for x in range(25):
+                     artist_df = pd.DataFrame([artist_id_uni[i+x]], columns=['Artist_uri'])
+                     artist_pop = artist_features['artists'][x]["popularity"]
+                     artist_genres = artist_features['artists'][x]["genres"]
+                     artist_df["Artist_pop"] = artist_pop
+                     if artist_genres:
+                         artist_df["genres"] = " ".join([re.sub(' ', '_', i) for i in artist_genres])
+                     else:
+                         artist_df["genres"] = "unknown"
+                     artist_ = pd.concat([artist_, artist_df], axis=0)
+             except Exception as e:
+                 err.append(e)
+                 continue
+         try:
+             test = pd.DataFrame(
+                 track_, columns=['Track_uri', 'Artist_uri', 'Album_uri'])
+
+             test.rename(columns={'Track_uri': 'track_uri',
+                         'Artist_uri': 'artist_uri', 'Album_uri': 'album_uri'}, inplace=True)
+
+             audio_features.drop(
+                 columns=['type', 'uri', 'track_href', 'analysis_url'], axis=1, inplace=True)
+
+             test = pd.merge(test, audio_features,
+                             left_on="track_uri", right_on="id", how='outer')
+             test = pd.merge(test, track_, left_on="track_uri",
+                             right_on="Track_uri", how='outer')
+             test = pd.merge(test, artist_, left_on="artist_uri",
+                             right_on="Artist_uri", how='outer')
+
+             test.rename(columns={'genres': 'Artist_genres'}, inplace=True)
+
+             test.drop(columns=['Track_uri', 'Artist_uri_x',
+                       'Artist_uri_y', 'Album_uri', 'id'], axis=1, inplace=True)
+
+             test.dropna(axis=0, inplace=True)
+             test['Track_pop'] = test['Track_pop'].apply(lambda x: int(x/5))
+             test['Artist_pop'] = test['Artist_pop'].apply(lambda x: int(x/5))
+             test['Track_release_date'] = test['Track_release_date'].apply(lambda x: x.split('-')[0])
+             test['Track_release_date'] = test['Track_release_date'].astype('int16')
+             test['Track_release_date'] = test['Track_release_date'].apply(lambda x: int(x/50))
+
+             test[['danceability', 'energy', 'key', 'loudness', 'mode', 'speechiness', 'acousticness', 'instrumentalness', 'liveness', 'valence', 'tempo', 'time_signature']] = test[[
+                 'danceability', 'energy', 'key', 'loudness', 'mode', 'speechiness', 'acousticness', 'instrumentalness', 'liveness', 'valence', 'tempo', 'time_signature']].astype('float16')
+             test[['duration_ms']] = test[['duration_ms']].astype('float32')
+             test[['Track_release_date', 'Track_pop', 'Artist_pop']] = test[[
+                 'Track_release_date', 'Track_pop', 'Artist_pop']].astype('int8')
+         except Exception as e:
+             err.append(e)
+         err.append('Finish extraction')
+         return test, err
+     test, err = extract(track_ids_uni, artist_id_uni)
+    
+     for i in err:
+         log.append(i)
+     del err
+     grow = test.copy()
+     test['Artist_genres'] = test['Artist_genres'].apply(lambda x: x.split(" "))
+     tfidf = TfidfVectorizer(max_features=max_gen)  
+     tfidf_matrix = tfidf.fit_transform(test['Artist_genres'].apply(lambda x: " ".join(x)))
+     genre_df = pd.DataFrame(tfidf_matrix.toarray())
+     genre_df.columns = ['genre' + "|" +i for i in tfidf.get_feature_names_out()]
+     genre_df = genre_df.astype('float16')
+     test.drop(columns=['Artist_genres'], axis=1, inplace=True)
+     test = pd.concat([test.reset_index(drop=True),genre_df.reset_index(drop=True)], axis=1)
+     Fresult = pd.DataFrame()
+     x = 1
+     for i in range(int(lendf/2), lendf+1, int(lendf/2)):
+         try:
+             df = pd.read_csv('Data/streamlit.csv',names= col_name,dtype=dtypes,skiprows=x,nrows=i)
+             log.append('reading data frame chunks from {} to {}'.format(x,i))
+         except Exception as e:
+             log.append('Failed to load grow')
+             log.append(e)
+         grow = grow[~grow['track_uri'].isin(df['track_uri'].values)]
+         df = df[~df['track_uri'].isin(test['track_uri'].values)]
+         df['Artist_genres'] = df['Artist_genres'].apply(lambda x: x.split(" "))
+         tfidf_matrix = tfidf.transform(df['Artist_genres'].apply(lambda x: " ".join(x)))
+         genre_df = pd.DataFrame(tfidf_matrix.toarray())
+         genre_df.columns = ['genre' + "|" +i for i in tfidf.get_feature_names_out()]
+         genre_df = genre_df.astype('float16')
+         df.drop(columns=['Artist_genres'], axis=1, inplace=True)
+         df = pd.concat([df.reset_index(drop=True),
+                        genre_df.reset_index(drop=True)], axis=1)
+         del genre_df
+         try:
+             df.drop(columns=['genre|unknown'], axis=1, inplace=True)
+             test.drop(columns=['genre|unknown'], axis=1, inplace=True)
+         except:
+             log.append('genre|unknown not found')
+         log.append('Scaling the data .....')
+         if x == 1:
+             sc = pickle.load(open('Data/sc.sav','rb'))
+             df.iloc[:, 3:19] = sc.transform(df.iloc[:, 3:19])
+             test.iloc[:, 3:19] = sc.transform(test.iloc[:, 3:19])
+             log.append("Creating playlist vector")
+             playvec = pd.DataFrame(test.sum(axis=0)).T
+         else:
+             df.iloc[:, 3:19] = sc.transform(df.iloc[:, 3:19])
+         x = i
+         if model == 'Model 1':
+             df['sim']=cosine_similarity(df.drop(['track_uri', 'artist_uri', 'album_uri'], axis = 1),playvec.drop(['track_uri', 'artist_uri', 'album_uri'], axis = 1))
+             df['sim2']=cosine_similarity(df.iloc[:,16:-1],playvec.iloc[:,16:])
+             df['sim3']=cosine_similarity(df.iloc[:,19:-2],playvec.iloc[:,19:])
+             df = df.sort_values(['sim3','sim2','sim'],ascending = False,kind='stable').groupby('artist_uri').head(same_art).head(50)
+             Fresult = pd.concat([Fresult, df], axis=0)
+             Fresult = Fresult.sort_values(['sim3', 'sim2', 'sim'],ascending=False,kind='stable')
+             Fresult.drop_duplicates(subset=['track_uri'], inplace=True,keep='first')
+             Fresult = Fresult.groupby('artist_uri').head(same_art).head(50)
+         elif model == 'Model 2':
+             df['sim'] = cosine_similarity(df.iloc[:, 3:16], playvec.iloc[:, 3:16])
+             df['sim2'] = cosine_similarity(df.loc[:, df.columns.str.startswith('T') | df.columns.str.startswith('A')], playvec.loc[:, playvec.columns.str.startswith('T') | playvec.columns.str.startswith('A')])
+             df['sim3'] = cosine_similarity(df.loc[:, df.columns.str.startswith('genre')], playvec.loc[:, playvec.columns.str.startswith('genre')])
+             df['sim4'] = (df['sim']+df['sim2']+df['sim3'])/3
+             df = df.sort_values(['sim4'], ascending=False,kind='stable').groupby('artist_uri').head(same_art).head(50)
+             Fresult = pd.concat([Fresult, df], axis=0)
+             Fresult = Fresult.sort_values(['sim4'], ascending=False,kind='stable')
+             Fresult.drop_duplicates(subset=['track_uri'], inplace=True,keep='first')
+             Fresult = Fresult.groupby('artist_uri').head(same_art).head(50)
+     del test
+     try:
+      del df
+      log.append('Getting Result')
+     except:
+         log.append('Getting Result')
+     if model == 'Model 1':
+         Fresult = Fresult.sort_values(['sim3', 'sim2', 'sim'],ascending=False,kind='stable')
+         Fresult.drop_duplicates(subset=['track_uri'], inplace=True,keep='first')
+         Fresult = Fresult.groupby('artist_uri').head(same_art).track_uri.head(50)
+     elif model == 'Model 2':
+         Fresult = Fresult.sort_values(['sim4'], ascending=False,kind='stable')
+         Fresult.drop_duplicates(subset=['track_uri'], inplace=True,keep='first')
+         Fresult = Fresult.groupby('artist_uri').head(same_art).track_uri.head(50)
+     log.append('{} New Tracks Found'.format(len(grow)))
+     if(len(grow)>=1):
+        try:
+            new=pd.read_csv('Data/new_tracks.csv',dtype=dtypes)
+            new=pd.concat([new, grow], axis=0)
+            new=new[new.Track_pop >0]
+            new.drop_duplicates(subset=['track_uri'], inplace=True,keep='last')
+            new.to_csv('Data/new_tracks.csv',index=False)
+        except:
+            grow.to_csv('Data/new_tracks.csv', index=False)
+     log.append('Model run successfully')
+    except Exception as e:
+        log.append("Model Failed")
+        log.append(e)
+    return Fresult, log
+
+
+
+def top_tracks(url,region):
+    uri = url.split('/')[-1].split('?')[0]
+    stream= open("Spotify/Spotify.yaml")
+    spotify_details = yaml.safe_load(stream)
+    auth_manager = SpotifyClientCredentials(client_id=spotify_details['Client_id'],client_secret=spotify_details['client_secret'])
+    sp = spotipy.client.Spotify(auth_manager=auth_manager)
+    log = []
+    Fresult = []
+    try:
+        log.append('Starting Spotify Model')
+        top=sp.artist_top_tracks(uri,country=region)
+        for i in range(10) :
+            Fresult.append(top['tracks'][i]['id'])
+        log.append('Model run successfully')
+    except Exception as e:
+        log.append("Model Failed")
+        log.append(e)
+    return Fresult,log
+
+def song_model(url, model, max_gen=3, same_art=5):
+    log = []
+    Fresult = []
+    try:
+     log.append('Start logging')
+     uri = url.split('/')[-1].split('?')[0]
+     stream = open("Spotify/Spotify.yaml")
+     spotify_details = yaml.safe_load(stream)
+     auth_manager = SpotifyClientCredentials(
+         client_id=spotify_details['Client_id'], client_secret=spotify_details['client_secret'])
+     sp = spotipy.client.Spotify(auth_manager=auth_manager) 
+
+     if model == 'Spotify Model':
+        log.append('Starting Spotify Model')
+        aa=sp.recommendations(seed_tracks=[uri], limit=25)
+        for i in range(25):
+            Fresult.append(aa['tracks'][i]['id'])
+        log.append('Model run successfully')
+        return Fresult, log
+     lendf=len(pd.read_csv('Data/streamlit.csv',usecols=['track_uri']))
+     dtypes = {'track_uri': 'object', 'artist_uri': 'object', 'album_uri': 'object', 'danceability': 'float16', 'energy': 'float16', 'key': 'float16',
+               'loudness': 'float16', 'mode': 'float16', 'speechiness': 'float16', 'acousticness': 'float16', 'instrumentalness': 'float16',
+               'liveness': 'float16', 'valence': 'float16', 'tempo': 'float16', 'duration_ms': 'float32', 'time_signature': 'float16',
+               'Track_release_date': 'int8', 'Track_pop': 'int8', 'Artist_pop': 'int8', 'Artist_genres': 'object'}
+     col_name= ['track_uri', 'artist_uri', 'album_uri', 'danceability', 'energy', 'key',
+        'loudness', 'mode', 'speechiness', 'acousticness', 'instrumentalness',
+        'liveness', 'valence', 'tempo', 'duration_ms', 'time_signature',
+        'Track_release_date', 'Track_pop', 'Artist_pop', 'Artist_genres']
+     log.append('Start audio features extraction')
+     audio_features = pd.DataFrame(sp.audio_features([uri]))
+     log.append('Start track features extraction')
+     track_ = pd.DataFrame()
+     track_features = sp.tracks([uri])
+     track_pop = pd.DataFrame([uri], columns=['Track_uri'])
+     track_pop['Track_release_date'] = track_features['tracks'][0]['album']['release_date']
+     track_pop['Track_pop'] = track_features['tracks'][0]["popularity"]
+     track_pop['Artist_uri'] = track_features['tracks'][0]['artists'][0]['id']
+     track_pop['Album_uri'] = track_features['tracks'][0]['album']['id']
+     track_ = pd.concat([track_, track_pop], axis=0)
+     log.append('Start artist features extraction')
+     artist_id_uni=list(track_['Artist_uri'])
+     artist_ = pd.DataFrame()
+     artist_features = sp.artists(artist_id_uni)
+     artist_df = pd.DataFrame(artist_id_uni, columns=['Artist_uri'])
+     artist_pop = artist_features['artists'][0]["popularity"]
+     artist_genres = artist_features['artists'][0]["genres"]
+     artist_df["Artist_pop"] = artist_pop
+     if artist_genres:
+        artist_df["genres"] = " ".join([re.sub(' ', '_', i) for i in artist_genres])
+     else:
+        artist_df["genres"] = "unknown"
+     artist_ = pd.concat([artist_, artist_df], axis=0)
+     try:
+        test = pd.DataFrame(track_, columns=['Track_uri', 'Artist_uri', 'Album_uri'])
+        test.rename(columns={'Track_uri': 'track_uri','Artist_uri': 'artist_uri', 'Album_uri': 'album_uri'}, inplace=True)
+        audio_features.drop(columns=['type', 'uri', 'track_href', 'analysis_url'], axis=1, inplace=True)
+        test = pd.merge(test, audio_features,left_on="track_uri", right_on="id", how='outer')
+        test = pd.merge(test, track_, left_on="track_uri",right_on="Track_uri", how='outer')
+        test = pd.merge(test, artist_, left_on="artist_uri",right_on="Artist_uri", how='outer')
+        test.rename(columns={'genres': 'Artist_genres'}, inplace=True)
+        test.drop(columns=['Track_uri', 'Artist_uri_x','Artist_uri_y', 'Album_uri', 'id'], axis=1, inplace=True)
+        test.dropna(axis=0, inplace=True)
+        test['Track_pop'] = test['Track_pop'].apply(lambda x: int(x/5))
+        test['Artist_pop'] = test['Artist_pop'].apply(lambda x: int(x/5))
+        test['Track_release_date'] = test['Track_release_date'].apply(lambda x: x.split('-')[0])
+        test['Track_release_date'] = test['Track_release_date'].astype('int16')
+        test['Track_release_date'] = test['Track_release_date'].apply(lambda x: int(x/50))
+        test[['danceability', 'energy', 'key', 'loudness', 'mode', 'speechiness', 'acousticness', 'instrumentalness', 'liveness', 'valence', 'tempo', 'time_signature']] = test[['danceability', 'energy', 'key', 'loudness', 'mode', 'speechiness', 'acousticness', 'instrumentalness', 'liveness', 'valence', 'tempo', 'time_signature']].astype('float16')
+        test[['duration_ms']] = test[['duration_ms']].astype('float32')
+        test[['Track_release_date', 'Track_pop', 'Artist_pop']] = test[['Track_release_date', 'Track_pop', 'Artist_pop']].astype('int8')
+     except Exception as e:
+        log.append(e)
+     log.append('Finish extraction')
+     grow = test.copy()
+     test['Artist_genres'] = test['Artist_genres'].apply(lambda x: x.split(" "))
+     tfidf = TfidfVectorizer(max_features=max_gen)  
+     tfidf_matrix = tfidf.fit_transform(test['Artist_genres'].apply(lambda x: " ".join(x)))
+     genre_df = pd.DataFrame(tfidf_matrix.toarray())
+     genre_df.columns = ['genre' + "|" +i for i in tfidf.get_feature_names_out()]
+     genre_df = genre_df.astype('float16')
+     test.drop(columns=['Artist_genres'], axis=1, inplace=True)
+     test = pd.concat([test.reset_index(drop=True),genre_df.reset_index(drop=True)], axis=1)
+     Fresult = pd.DataFrame()
+     x = 1
+     for i in range(int(lendf/2), lendf+1, int(lendf/2)):
+         try:
+             df = pd.read_csv('Data/streamlit.csv',names= col_name,dtype=dtypes,skiprows=x,nrows=i)
+             log.append('reading data frame chunks from {} to {}'.format(x,i))
+         except Exception as e:
+             log.append('Failed to load grow')
+             log.append(e)
+         grow = grow[~grow['track_uri'].isin(df['track_uri'].values)]
+         df = df[~df['track_uri'].isin(test['track_uri'].values)]
+         df['Artist_genres'] = df['Artist_genres'].apply(lambda x: x.split(" "))
+         tfidf_matrix = tfidf.transform(df['Artist_genres'].apply(lambda x: " ".join(x)))
+         genre_df = pd.DataFrame(tfidf_matrix.toarray())
+         genre_df.columns = ['genre' + "|" +i for i in tfidf.get_feature_names_out()]
+         genre_df = genre_df.astype('float16')
+         df.drop(columns=['Artist_genres'], axis=1, inplace=True)
+         df = pd.concat([df.reset_index(drop=True),
+                        genre_df.reset_index(drop=True)], axis=1)
+         del genre_df
+         try:
+             df.drop(columns=['genre|unknown'], axis=1, inplace=True)
+             test.drop(columns=['genre|unknown'], axis=1, inplace=True)
+         except:
+             log.append('genre|unknown not found')
+         log.append('Scaling the data .....')
+         if x == 1:
+             sc = pickle.load(open('Data/sc.sav','rb'))
+             df.iloc[:, 3:19] = sc.transform(df.iloc[:, 3:19])
+             test.iloc[:, 3:19] = sc.transform(test.iloc[:, 3:19])
+             log.append("Creating playlist vector")
+             playvec = pd.DataFrame(test.sum(axis=0)).T
+         else:
+             df.iloc[:, 3:19] = sc.transform(df.iloc[:, 3:19])
+         x = i
+         if model == 'Model 1':
+             df['sim']=cosine_similarity(df.drop(['track_uri', 'artist_uri', 'album_uri'], axis = 1),playvec.drop(['track_uri', 'artist_uri', 'album_uri'], axis = 1))
+             df['sim2']=cosine_similarity(df.iloc[:,16:-1],playvec.iloc[:,16:])
+             df['sim3']=cosine_similarity(df.iloc[:,19:-2],playvec.iloc[:,19:])
+             df = df.sort_values(['sim3','sim2','sim'],ascending = False,kind='stable').groupby('artist_uri').head(same_art).head(50)
+             Fresult = pd.concat([Fresult, df], axis=0)
+             Fresult = Fresult.sort_values(['sim3', 'sim2', 'sim'],ascending=False,kind='stable')
+             Fresult.drop_duplicates(subset=['track_uri'], inplace=True,keep='first')
+             Fresult = Fresult.groupby('artist_uri').head(same_art).head(50)
+         elif model == 'Model 2':
+             df['sim'] = cosine_similarity(df.iloc[:, 3:16], playvec.iloc[:, 3:16])
+             df['sim2'] = cosine_similarity(df.loc[:, df.columns.str.startswith('T') | df.columns.str.startswith('A')], playvec.loc[:, playvec.columns.str.startswith('T') | playvec.columns.str.startswith('A')])
+             df['sim3'] = cosine_similarity(df.loc[:, df.columns.str.startswith('genre')], playvec.loc[:, playvec.columns.str.startswith('genre')])
+             df['sim4'] = (df['sim']+df['sim2']+df['sim3'])/3
+             df = df.sort_values(['sim4'], ascending=False,kind='stable').groupby('artist_uri').head(same_art).head(50)
+             Fresult = pd.concat([Fresult, df], axis=0)
+             Fresult = Fresult.sort_values(['sim4'], ascending=False,kind='stable')
+             Fresult.drop_duplicates(subset=['track_uri'], inplace=True,keep='first')
+             Fresult = Fresult.groupby('artist_uri').head(same_art).head(50)
+     del test
+     try:
+      del df
+      log.append('Getting Result')
+     except:
+         log.append('Getting Result')
+     if model == 'Model 1':
+         Fresult = Fresult.sort_values(['sim3', 'sim2', 'sim'],ascending=False,kind='stable')
+         Fresult.drop_duplicates(subset=['track_uri'], inplace=True,keep='first')
+         Fresult = Fresult.groupby('artist_uri').head(same_art).track_uri.head(50)
+     elif model == 'Model 2':
+         Fresult = Fresult.sort_values(['sim4'], ascending=False,kind='stable')
+         Fresult.drop_duplicates(subset=['track_uri'], inplace=True,keep='first')
+         Fresult = Fresult.groupby('artist_uri').head(same_art).track_uri.head(50)
+     log.append('{} New Tracks Found'.format(len(grow)))
+     if(len(grow)>=1):
+        try:
+            new=pd.read_csv('Data/new_tracks.csv',dtype=dtypes)
+            new=pd.concat([new, grow], axis=0)
+            new=new[new.Track_pop >0]
+            new.drop_duplicates(subset=['track_uri'], inplace=True,keep='last')
+            new.to_csv('Data/new_tracks.csv',index=False)
+        except:
+            grow.to_csv('Data/new_tracks.csv', index=False)
+     log.append('Model run successfully')
+    except Exception as e:
+        log.append("Model Failed")
+        log.append(e)
+    return Fresult, log
+
+def update_dataset():
+    col_name= ['track_uri', 'artist_uri', 'album_uri', 'danceability', 'energy', 'key',
+        'loudness', 'mode', 'speechiness', 'acousticness', 'instrumentalness',
+        'liveness', 'valence', 'tempo', 'duration_ms', 'time_signature',
+        'Track_release_date', 'Track_pop', 'Artist_pop', 'Artist_genres']
+    dtypes = {'track_uri': 'object', 'artist_uri': 'object', 'album_uri': 'object', 'danceability': 'float16', 'energy': 'float16', 'key': 'float16',
+               'loudness': 'float16', 'mode': 'float16', 'speechiness': 'float16', 'acousticness': 'float16', 'instrumentalness': 'float16',
+               'liveness': 'float16', 'valence': 'float16', 'tempo': 'float16', 'duration_ms': 'float32', 'time_signature': 'float16',
+               'Track_release_date': 'int8', 'Track_pop': 'int8', 'Artist_pop': 'int8', 'Artist_genres': 'object'}
+    df = pd.read_csv('Data/streamlit.csv',dtype=dtypes)
+    grow = pd.read_csv('Data/new_tracks.csv',dtype=dtypes)
+    cur = len(df)
+    df=pd.concat([df,grow],axis=0)
+    grow=pd.DataFrame(columns=col_name)
+    grow.to_csv('Data/new_tracks.csv',index=False)
+    df=df[df.Track_pop >0]
+    df.drop_duplicates(subset=['track_uri'],inplace=True,keep='last')
+    df.dropna(axis=0,inplace=True)
+    df.to_csv('Data/streamlit.csv',index=False)
+    return (len(df)-cur)
+

requirements.txt

@@ -0,0 +1,6 @@
+pandas==1.4.4
+PyYAML==6.0
+scikit_learn==1.1.3
+spotipy==2.20.0
+streamlit==1.14.0
+streamlit_option_menu==0.3.2