<jupyter_start><jupyter_text>raw data<jupyter_code>df = pd.read_pickle('../data/songs_counts_200.pkl')
# df = df[:550]
df
len(df)<jupyter_output><empty_output><jupyter_text>track features<jupyter_code># define batches
batch_size = 100
num_batches = math.ceil(len(df)/batch_size)

# initialize list to save API calls
track_features = []

start_time = time.time()

# looping through the batches
for i in range(num_batches):
    
    
    # define start and end of the batch
    start_point = i*batch_size
    end_point = min(start_point + batch_size, len(df))
    
    # API call
    track_list = list(df['track_uri'][start_point:end_point])
    track_features.extend(sp.audio_features(track_list))

    if i%100 == 0:
        print('{}/{}, {}s'.format(i, num_batches, time.time()-start_time))
        start_time = time.time()
    
# track_features = [i for i in track_features if i is not None]
track_features_df = pd.DataFrame(track_features)
track_features_df
# counter = 0
track_features_df.to_csv('../data/track_features'+str(counter)+'.csv')
track_features_df.to_pickle('../data/track_features'+str(counter)+'.pkl')
counter += 1<jupyter_output><empty_output><jupyter_text>artist info<jupyter_code>unique_artists = list(df['artist_uri'].unique())
len(unique_artists)
# define batches
batch_size = 50
num_batches = math.ceil(len(unique_artists)/batch_size)

# initialize list to save API calls
artist_info = []

start_time = time.time()

# looping through the batches
for i in range(num_batches):
    
    
    # define start and end of the batch
    start_point = i*batch_size
    end_point = min(start_point + batch_size, len(df))
    
    # API call
    artist_list = unique_artists[start_point:end_point]
    artist_info.extend(sp.artists(artist_list)['artists'])

    if i%100 == 0:
        print('{}/{}, {}s'.format(i, num_batches, time.time()-start_time))
        start_time = time.time()
    
# artist_info = [i for i in artist_info if i is not None]
artist_info_df = pd.DataFrame(artist_info)
artist_info_df
len(set(list(chain.from_iterable(artist_info_df['genres']))))
counter = 0
# path = '../data/artist_info'+str(counter2)+'.pkl'
# with open(path, 'wb') as file:
#     pickle.dump(artist_info, file)

# counter2 += 1
# with open(path, 'rb') as file:
#     artist_data = pickle.load(file)
artist_info_df.to_csv('../data/artist_info'+str(counter)+'.csv')
artist_info_df.to_pickle('../data/artist_info'+str(counter)+'.pkl')
counter += 1
# counter<jupyter_output><empty_output><jupyter_text>album info<jupyter_code>unique_albums = list(df['album_uri'].unique())
len(unique_albums)
# albums_exist = pd.read_pickle('../data/album_info1.pkl')
albums_exist = list(albums_exist['uri'])
print(len(albums_exist))
# albums_exist
len(set(unique_albums_new))
# unique_albums_new = [i for i in unique_albums if i not in albums_exist]
unique_albums_new = list(set(unique_albums) - set(albums_exist))
print(len(unique_albums_new))
# unique_albums_new
unique_albums = unique_albums_new[140000:]
len(unique_albums)
# define batches
batch_size = 20
num_batches = math.ceil(len(unique_albums)/batch_size)

# initialize list to save API calls
album_info = []

start_time = time.time()

# looping through the batches
for i in range(num_batches):
    
    
    # define start and end of the batch
    start_point = i*batch_size
    end_point = min(start_point + batch_size, len(df))
    
    # API call
    album_list = unique_albums[start_point:end_point]
    album_info.extend(sp.albums(album_list)['albums'])

    if i%100 == 0:
        print('{}/{}, {}s'.format(i, num_batches, time.time()-start_time))
        start_time = time.time()
    
album_info = [i for i in album_info if i is not None]
album_info_df = pd.DataFrame(album_info)
album_info_df
# counter = 4
album_info_df.to_csv('../data/album_info'+str(counter)+'.csv')
album_info_df.to_pickle('../data/album_info'+str(counter)+'.pkl')
counter += 1
counter<jupyter_output><empty_output><jupyter_text>data joining<jupyter_code>album_columns = ['genres','popularity','release_date','uri']

albums1 = pd.read_csv('../data/album_info1.csv', usecols=album_columns)
albums2 = pd.read_csv('../data/album_info2.csv', usecols=album_columns)
albums3 = pd.read_csv('../data/album_info3.csv', usecols=album_columns)
albums4 = pd.read_csv('../data/album_info4.csv', usecols=album_columns)
albums5 = pd.read_csv('../data/album_info5.csv', usecols=album_columns)
albums6 = pd.read_csv('../data/album_info6.csv', usecols=album_columns)
albums = pd.concat([albums1, albums2, albums3, albums4, albums5, albums6], axis=0, ignore_index=True)

albums = albums.rename(columns={'genres': 'album_genres', 'popularity': 'album_popularity', 'release_date': 'album_release_date', 'uri': 'album_uri'})
albums = albums.drop_duplicates()

albums
artist_columns = ['genres','popularity','uri']
artists = pd.read_csv('../data/artist_info1.csv', usecols=artist_columns)

artists = artists.rename(columns={'genres': 'artist_genres', 'popularity': 'artist_popularity', 'uri': 'artist_uri'})
artists = artists.drop_duplicates()

artists
track_columns = ['danceability','energy','key','loudness','mode','speechiness','acousticness','instrumentalness','liveness','valence','tempo','time_signature','uri']
tracks = pd.read_csv('../data/track_features3.csv', usecols=track_columns)

tracks = tracks.rename(columns={'uri': 'track_uri'})
tracks = tracks.drop_duplicates()

tracks
master = pd.read_pickle('../data/songs_counts_200.pkl')
master['song_id'] = master.index
master
master = master.merge(track_features, on='track_uri', suffixes=(None, '_tracks'))
master = master.merge(artists, on='artist_uri', suffixes=(None, '_artists'))
master = master.merge(albums, on='album_uri', suffixes=(None, '_albums'))
master = master.set_index('song_id')
master
master.shape
master.to_csv('../data/master200.csv')
master.to_pickle('../data/master200.pkl')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1529051987767584, 'fraction_numerical': 0.012062521236833164, 'mean_word_length': 5.790080738177624, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '152410', 'license': 'MIT', 'n_tokens_mistral': 2196, 'n_tokens_neox': 2150, 'n_words': 364, 'path': 'milestone3/final_submission/spotify_api.ipynb', 'repo_name': 'johannes-kk/TheDigitalFrontier', 'size': '5886'}