<jupyter_start><jupyter_text>Get Spotify credentials<jupyter_code># Get required info for token request
import os
from dotenv import load_dotenv
load_dotenv()
client_id = os.getenv("SPOTIFY_CLIENT_ID")
client_secret = os.getenv("SPOTIFY_CLIENT_SECRET")
token_url = "https://accounts.spotify.com/api/token"

# Perform OAuth client credentials authorization
import requests
r = requests.post(
    "https://accounts.spotify.com/api/token",
    data={ "grant_type": "client_credentials" },
    auth=(client_id, client_secret)
)
r.raise_for_status()
response = r.json()
token = response["access_token"]
headers = { "Authorization": f"Bearer {token}" }<jupyter_output><empty_output><jupyter_text>Get Song a Day playlist data<jupyter_code>playlists = { 2019: "6xJjyO5AGrZkyGkWYXfUX8", 2020: "3iNn4yKx9PxpSVYfRXkK8i" }
years = sorted(playlists.keys())
tracks = { year: [] for year in years }

# For each year's playlist, grab tracks
for year in years:
    # Page through track list for the playlist
    next_url = f"https://api.spotify.com/v1/playlists/{playlists[year]}/tracks"
    while next_url:
        # Fetch the current page
        r = requests.get(next_url, headers=headers)
        r.raise_for_status()
        response = r.json()
        # Reshape each playlist item and add to our "tracks" list for the relevant year
        for playlist_item in response["items"]:
            img_sizes = { 640: "large", 300: "medium", 64: "small" }
            tracks[year].append({
                "track_name": playlist_item["track"]["name"],
                "track_id": playlist_item["track"]["id"],
                "album_id": playlist_item["track"]["album"]["id"],
                "album_name": playlist_item["track"]["album"]["name"],
                **dict([
                    ("album_image_{size}".format(size=img_sizes[i['width']]), i["url"])
                    for i in playlist_item["track"]["album"]["images"]
                ]),
                "artists": [
                    { "name": artist["name"], "id": artist["id"] }
                    for artist in playlist_item["track"]["artists"]
                ],
                "primary_artist_name": playlist_item["track"]["artists"][0]["name"],
                "primary_artist_id": playlist_item["track"]["artists"][0]["id"],
                "is_local": playlist_item["is_local"]
            })
        # Take note of the URL of the next page of playlist items
        next_url = response["next"]
# Compile all tracks
all_tracks = []
# Assign a date to each track
from datetime import datetime, timedelta
for year in years:
    curr_date = datetime(year, 1, 1)
    for track in tracks[year]:
        track["date"] = curr_date
        all_tracks.append(track) # add to big array
        curr_date += timedelta(days=1)

print(tracks[2019][-1])<jupyter_output>{'track_name': 'Northstar (feat. <NAME> & <NAME>)', 'track_id': '2rQo6riaUQ9KRzDgPLqeWx', 'album_id': '1H2Y4lomFedlGP6yEd15cT', 'album_name': 'Opia', 'album_image_large': 'https://i.scdn.co/image/ab67616d0000b2732fe16c52fdb5c238f66586c1', 'album_image_medium': 'https://i.scdn.co/image/ab67616d00001e022fe16c52fdb5c238f66586c1', 'album_image_small': 'https://i.scdn.co/image/ab67616d000048512fe16c52fdb5c238f66586c1', 'artists': [{'name': '<NAME>', 'id': '1ikfT4dtC0eCVk4TZdfHGO'}], 'primary_artist_name': '<NAME>', 'primary_artist_id': '1ikfT4dtC0eCVk4TZdfHGO', 'is_local': False, 'date': datetime.datetime(2019, 12, 31, 0, 0)}<jupyter_text>Add to Pandas<jupyter_code>import pandas as pd
pd.options.display.max_columns = None

track_data_df = pd.DataFrame(all_tracks)
track_data_df<jupyter_output><empty_output><jupyter_text>Get audio features for each track<jupyter_code>audio_features_data = []

# deduplicated ID of every track.
# This ensures that our audio_features dataframe only contains each track once and we can
# perform a many-to-one merge.
track_ids = list(set([track["track_id"] for track in all_tracks if not track["is_local"]]))

# Fetch data from spotify API
for idx in range(0, len(track_ids), 100):
    ids_batch = track_ids[idx:idx+100]
    endpoint = "https://api.spotify.com/v1/audio-features"
    r = requests.get(endpoint, headers=headers, params={ "ids": ",".join(ids_batch) })
    r.raise_for_status()
    keys = [
        "id",
        # Concrete musical properties
        "mode", "key", "tempo", "time_signature", "duration_ms",
        # Mood
        "danceability", "energy", "speechiness", "acousticness", "instrumentalness", "liveness", "valence"
    ]
    audio_features_data.extend([
        { k: track_data[k] for k in keys }
        for track_data in r.json()["audio_features"]
    ])

# Merge this new data into basic track data
audio_features_df = pd.DataFrame(audio_features_data).rename(columns={ "id" : "track_id" })
tracks_df = pd.merge(track_data_df, audio_features_df, how="left", on="track_id", validate="many_to_one")
tracks_df<jupyter_output><empty_output><jupyter_text>Export data<jupyter_code># Abbreviate
export_df = tracks_df.drop(columns=["album_image_medium", "artists"])

# Shorten image URLs to save space
def discard_cdn_prefix(url):
    prefix = "https://i.scdn.co/image/"
    return url[len(prefix):] if isinstance(url, str) and url.startswith(prefix) else url
export_df["album_image_large"] = export_df["album_image_large"].apply(discard_cdn_prefix)
export_df["album_image_small"] = export_df["album_image_small"].apply(discard_cdn_prefix)

# Date index makes JSON export nicer
export_df.set_index("date", inplace=True)

# Export data

# Formatted JSON for easy reading
export_df.to_json("./tracks.json", orient="index", date_format="iso", indent=2)
# CSV is the "clean" distributable data export; CSV takes up a lot less space than even non-indented JSON does
export_df.to_csv("./tracks.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13987581924801656, 'fraction_numerical': 0.026215936529837874, 'mean_word_length': 4.200896860986547, 'pattern_counts': {'":': 13, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '619166', 'license': 'MIT', 'n_tokens_mistral': 2092, 'n_tokens_neox': 1983, 'n_words': 419, 'path': 'src/stats.ipynb', 'repo_name': 'controversial/song-a-day', 'size': '5823'}