linked mlops from git dir & added evaluation notebook

mlops/notebooks/README.md

@@ -0,0 +1,8 @@
+# Jupyter Notebooks, Exploratory Analysis and Prototyping
+
+Jupyter colabs and python scripts in this directory are used to create a processing pipeline that help us generate data and load it into our project database. The cloud database chosen has data related to:
+
+1. User-to-Song rating/playback information
+2. Song-to-Song similarity information based on musicality.
+
+These two sets of data will be used to train machine learning models within the MLOps phase of our project.

mlops/notebooks/label_encoder.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cd6ec677ead34e520071f2936debeaf08153f6437798670b574a8425884aa74
+size 241117

mlops/notebooks/model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66c599a48aa1280d790daa9f7cff45271edb5952ec9d8401d53e470af322801c
+size 2772520

mlops/notebooks/model_training.ipynb

@@ -0,0 +1 @@
+{"cells":[{"cell_type":"markdown","metadata":{"id":"5ByvVHnFr-s1"},"source":["Get million song subset data song list\n","Get metadata and join the data\n","\n","use artist similarity and artists to train the model on similarity\n","\n","use last.fm to get additional data on each song to augment this"]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1048,"status":"ok","timestamp":1715387243880,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"dO4mavAdsELi","outputId":"d0064229-e1a9-4875-e8f1-ee2b19e36855"},"outputs":[{"ename":"ModuleNotFoundError","evalue":"No module named 'google.colab'","output_type":"error","traceback":["\u001b[1;31m---------------------------------------------------------------------------\u001b[0m","\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)","Cell \u001b[1;32mIn[2], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mgoogle\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcolab\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m drive\n\u001b[0;32m      2\u001b[0m drive\u001b[38;5;241m.\u001b[39mmount(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/content/drive\u001b[39m\u001b[38;5;124m'\u001b[39m)\n","\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'google.colab'"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive')"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":7109,"status":"ok","timestamp":1715387250988,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"RktUo1FTsTm4","outputId":"ef21ad0f-4c34-4693-f5de-e6f5617465d7"},"outputs":[{"name":"stdout","output_type":"stream","text":["Requirement already satisfied: python-dotenv in /usr/local/lib/python3.10/dist-packages (1.0.1)\n"]}],"source":["!pip install python-dotenv"]},{"cell_type":"code","execution_count":3,"metadata":{"executionInfo":{"elapsed":1591,"status":"ok","timestamp":1715387252577,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"AkHtP67Sr-s2"},"outputs":[],"source":["# imports\n","import pandas as pd\n","import h5py\n","import os\n","from sqlalchemy import create_engine\n","import requests\n","import time\n","from dotenv import load_dotenv"]},{"cell_type":"code","execution_count":4,"metadata":{"executionInfo":{"elapsed":3,"status":"ok","timestamp":1715387252577,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"vIJPMBNFr-s3"},"outputs":[],"source":["pd.set_option('display.max_rows', 100)"]},{"cell_type":"code","execution_count":5,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"elapsed":2,"status":"ok","timestamp":1715387252577,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"kShsPDUoW0Tm","outputId":"5c3cb25e-e38f-4d9d-9102-1e18fd73618a"},"outputs":[{"data":{"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"},"text/plain":["'/content'"]},"execution_count":5,"metadata":{},"output_type":"execute_result"}],"source":["os.getcwd()"]},{"cell_type":"code","execution_count":6,"metadata":{"executionInfo":{"elapsed":2,"status":"ok","timestamp":1715387252577,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"kmdF3bjKW1kz"},"outputs":[],"source":["os.chdir('/content/drive/MyDrive/CMPE-258: Team Neurobytes/Neurobytes/mlops/notebooks')"]},{"cell_type":"code","execution_count":7,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":318,"status":"ok","timestamp":1715387252893,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"JKeiMiYMWwpM","outputId":"1eee2de4-7144-45fb-9269-a7eed567bc2d"},"outputs":[{"name":"stdout","output_type":"stream","text":["label_encoder.joblib  model_training.ipynb  scaler.joblib\t    tracks_eda.ipynb\n","model.pth\t      README.md\t\t    test_spotify_api.ipynb  users_eda.ipynb\n"]}],"source":["! ls"]},{"cell_type":"markdown","metadata":{"id":"sk1jv62kr-s3"},"source":["# Loading Data"]},{"cell_type":"markdown","metadata":{"id":"rTCKoervr-s3"},"source":["## Loading million song subset data"]},{"cell_type":"code","execution_count":9,"metadata":{"executionInfo":{"elapsed":162,"status":"ok","timestamp":1715364589548,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"zdtzGTb4r-s3"},"outputs":[],"source":["# load the data (only loading song_id, metadata contains the rest)\n","def read_song_features(file_path):\n","    with h5py.File(file_path, 'r') as f:\n","        song_id = f['metadata']['songs']['song_id'][0].decode('utf-8')\n","        return {'song_id': song_id}\n","\n","\n","# process all files in a directory into a df\n","def process_all_files_to_dataframe(root_dir):\n","    data = []\n","    print(f\"Checking directory: {root_dir}\")\n","\n","    for subdir, dirs, files in os.walk(root_dir):\n","        print(f\"Currently scanning {subdir} with {len(files)} files\")\n","        for file in files:\n","            if file.endswith('.h5'):\n","                file_path = os.path.join(subdir, file)\n","                print(f\"Processing file: {file_path}\")\n","                song_data = read_song_features(file_path)\n","                data.append(song_data)\n","\n","    if not data:\n","        print(\"No data to process.\")\n","\n","    df = pd.DataFrame(data)\n","    return df"]},{"cell_type":"code","execution_count":11,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":163,"status":"ok","timestamp":1715364643278,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"4WPOjZXOr-s3","outputId":"8ebedd7f-7741-4d5f-d46b-8f8399ff1c54"},"outputs":[{"name":"stdout","output_type":"stream","text":["Checking directory: ../../../data/\n","No data to process.\n"]}],"source":["root_dir = 'data/MillionSongSubset'\n","df = process_all_files_to_dataframe(root_dir)"]},{"cell_type":"markdown","metadata":{"id":"cM5Cf9MEr-s3"},"source":["### Loading million song subset metadata from sqlite db"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"JBos93r4r-s3"},"outputs":[],"source":["# load metadata from sqlite\n","def load_data_from_sqlite(db_path, table_name):\n","    engine = create_engine(f'sqlite:///{db_path}')\n","    query = f\"SELECT * FROM {table_name}\"\n","    df = pd.read_sql_query(query, engine)\n","    return df\n","\n","# load metadata and merge with song data\n","db_path3 = 'data/MillionSongSubsetMetadata/track_metadata.db'\n","df3 = load_data_from_sqlite(db_path3, 'songs')\n","df = df.merge(df3, on='song_id', how='left')\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"B2zIZ4T6r-s4"},"outputs":[],"source":["columns_to_drop = ['track_id', 'artist_id', 'song_id', 'artist_mbid', 'track_7digitalid', 'shs_perf', 'shs_work']\n","\n","for column in columns_to_drop:\n","    if column in df.columns:\n","        df.drop(columns=[column], inplace=True)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"NWP5gZpzr-s4"},"outputs":[],"source":["df.columns"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"TcgnhEffr-s4"},"outputs":[],"source":["df.head()"]},{"cell_type":"markdown","metadata":{"id":"9XEtFgNrr-s4"},"source":["## Loading last.fm data"]},{"cell_type":"code","execution_count":8,"metadata":{"executionInfo":{"elapsed":214,"status":"ok","timestamp":1715387284033,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"ce8MTL8dr-s4"},"outputs":[],"source":["def fetch_data(api_key, method, params):\n","    base_url = \"http://ws.audioscrobbler.com/2.0/\"\n","    params['api_key'] = api_key\n","    params['method'] = method\n","    params['format'] = 'json'\n","    response = requests.get(base_url, params=params)\n","    return response.json()\n","\n","\n","def get_artist_info(api_key, artist_name):\n","    params = {'artist': artist_name}\n","    return fetch_data(api_key, 'artist.getInfo', params)\n","\n","\n","def get_track_info(api_key, artist_name, track_name):\n","    params = {'artist': artist_name, 'track': track_name}\n","    return fetch_data(api_key, 'track.getInfo', params)\n","\n","\n","def batch_fetch_data(api_key, items, fetch_function, sleep_time=1):\n","    results = []\n","    for item in items:\n","        result = fetch_function(api_key, *item)\n","        results.append(result)\n","        # time.sleep(sleep_time)\n","    return results"]},{"cell_type":"code","execution_count":9,"metadata":{"executionInfo":{"elapsed":1,"status":"ok","timestamp":1715387284241,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"4NqoJJOYr-s4"},"outputs":[],"source":["# load LASTFM_API_KEY from .env\n","import requests\n","load_dotenv()\n","api_key = os.getenv('LASTFM_API_KEY')\n","\n","\n","def fetch_lastfm_data(api_key, artist_name, track_name):\n","    base_url = \"http://ws.audioscrobbler.com/2.0/\"\n","    params = {\n","        'method': 'track.getInfo',\n","        'api_key': api_key,\n","        'artist': artist_name,\n","        'track': track_name,\n","        'format': 'json'\n","    }\n","    response = requests.get(base_url, params=params)\n","    if response.status_code == 200 and response.text.strip():\n","        return response.json()\n","    else:\n","        return None\n","\n","\n","def parse_lastfm_data(data):\n","    if data and 'track' in data:\n","        track = data['track']\n","        return {\n","            'listeners': track.get('listeners', '0'),\n","            'playcount': track.get('playcount', '0'),\n","            'tags': ', '.join(tag['name'] for tag in track.get('toptags', {}).get('tag', [])),\n","        }\n","    return None"]},{"cell_type":"code","execution_count":10,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":216},"executionInfo":{"elapsed":357,"status":"error","timestamp":1715387298868,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"prQYTEpGr-s4","outputId":"7f04e536-cd64-41d8-861f-1ef035e99cab"},"outputs":[{"ename":"NameError","evalue":"name 'df' is not defined","output_type":"error","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m<ipython-input-10-da14a3af7ef7>\u001b[0m in \u001b[0;36m<cell line: 6>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mload_dotenv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mapi_key\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetenv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'LASTFM_API_KEY'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0msubset_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mtracks_skipped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mNameError\u001b[0m: name 'df' is not defined"]}],"source":["from tqdm import tqdm\n","tqdm.pandas()\n","\n","load_dotenv()\n","api_key = os.getenv('LASTFM_API_KEY')\n","subset_df = df.head(1000)\n","\n","tracks_skipped = 0\n","\n","\n","def fetch_and_parse(row):\n","    global tracks_skipped\n","    data = fetch_lastfm_data(api_key, row['artist_name'], row['title'])\n","    if data is None:\n","        tracks_skipped += 1\n","        return None\n","    parsed_data = parse_lastfm_data(data)\n","    if parsed_data is None:\n","        tracks_skipped += 1\n","    return parsed_data\n","\n","\n","# Use progress_apply instead of apply\n","subset_df['lastfm_data'] = subset_df.progress_apply(fetch_and_parse, axis=1)\n","\n","# Remove rows where lastfm_data is None\n","subset_df = subset_df[subset_df['lastfm_data'].notna()]\n","\n","subset_df.reset_index(drop=True, inplace=True)\n","track_details_df = pd.json_normalize(subset_df['lastfm_data'])\n","mixed = pd.concat(\n","    [subset_df.drop(columns=['lastfm_data']), track_details_df], axis=1)\n","\n","print(f\"Tracks skipped: {tracks_skipped}\")\n","\n","mixed.to_csv('data/music_data_small.csv', index=False)"]},{"cell_type":"markdown","metadata":{"id":"0wxV_-P6r-s5"},"source":["## Data processing"]},{"cell_type":"code","execution_count":191,"metadata":{"executionInfo":{"elapsed":191,"status":"ok","timestamp":1715390440593,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"vMUDiJbjr-s5"},"outputs":[],"source":["import pandas as pd\n","\n","df = pd.read_csv(\"..\\..\\db\\data\\music_data.csv\")\n","df.dropna(inplace=True)"]},{"cell_type":"code","execution_count":192,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":258},"executionInfo":{"elapsed":3,"status":"ok","timestamp":1715390440851,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"gHQ3NQr7rBN4","outputId":"6897914c-b362-4ea0-94e0-a5030f95fe88"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>release</th>\n","      <th>artist_name</th>\n","      <th>duration</th>\n","      <th>artist_familiarity</th>\n","      <th>artist_hotttnesss</th>\n","      <th>year</th>\n","      <th>listeners</th>\n","      <th>playcount</th>\n","      <th>tags</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>100 Club 1996 ''We Love You Beatles'' - Live</td>\n","      <td>Sex Pistols - The Interviews</td>\n","      <td>Sex Pistols</td>\n","      <td>88.73751</td>\n","      <td>0.731184</td>\n","      <td>0.549204</td>\n","      <td>0</td>\n","      <td>172</td>\n","      <td>210</td>\n","      <td>The Beatles, title is a full sentence</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Yo Quiero Contigo</td>\n","      <td>Sentenciados - Platinum Edition</td>\n","      <td>Baby Rasta &amp; Gringo</td>\n","      <td>167.36608</td>\n","      <td>0.610186</td>\n","      <td>0.355320</td>\n","      <td>0</td>\n","      <td>9753</td>\n","      <td>16911</td>\n","      <td>Reggaeton, alexis y fido, Eliana, mis videos, ...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Emerald</td>\n","      <td>Emerald</td>\n","      <td>Bedrock</td>\n","      <td>501.86404</td>\n","      <td>0.654039</td>\n","      <td>0.390625</td>\n","      <td>2004</td>\n","      <td>973</td>\n","      <td>2247</td>\n","      <td>dance</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>Karma</td>\n","      <td>The Diary Of Alicia Keys</td>\n","      <td>Alicia Keys</td>\n","      <td>255.99955</td>\n","      <td>0.933916</td>\n","      <td>0.778674</td>\n","      <td>2003</td>\n","      <td>250304</td>\n","      <td>1028356</td>\n","      <td>rnb, soul, Alicia Keys, female vocalists, Karma</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>Money Blues</td>\n","      <td>Slidetime</td>\n","      <td>Joanna Connor</td>\n","      <td>243.66975</td>\n","      <td>0.479218</td>\n","      <td>0.332857</td>\n","      <td>0</td>\n","      <td>429</td>\n","      <td>1008</td>\n","      <td>guitar girl, blues</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["                                          title  \\\n","0  100 Club 1996 ''We Love You Beatles'' - Live   \n","1                             Yo Quiero Contigo   \n","4                                       Emerald   \n","6                                         Karma   \n","7                                   Money Blues   \n","\n","                           release          artist_name   duration  \\\n","0     Sex Pistols - The Interviews          Sex Pistols   88.73751   \n","1  Sentenciados - Platinum Edition  Baby Rasta & Gringo  167.36608   \n","4                          Emerald              Bedrock  501.86404   \n","6         The Diary Of Alicia Keys          Alicia Keys  255.99955   \n","7                        Slidetime        Joanna Connor  243.66975   \n","\n","   artist_familiarity  artist_hotttnesss  year  listeners  playcount  \\\n","0            0.731184           0.549204     0        172        210   \n","1            0.610186           0.355320     0       9753      16911   \n","4            0.654039           0.390625  2004        973       2247   \n","6            0.933916           0.778674  2003     250304    1028356   \n","7            0.479218           0.332857     0        429       1008   \n","\n","                                                tags  \n","0              The Beatles, title is a full sentence  \n","1  Reggaeton, alexis y fido, Eliana, mis videos, ...  \n","4                                              dance  \n","6    rnb, soul, Alicia Keys, female vocalists, Karma  \n","7                                 guitar girl, blues  "]},"execution_count":192,"metadata":{},"output_type":"execute_result"}],"source":["df.head()"]},{"cell_type":"code","execution_count":193,"metadata":{"executionInfo":{"elapsed":142,"status":"ok","timestamp":1715390441226,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"ltW1gvc7r-s5"},"outputs":[],"source":["import pandas as pd\n","import torch\n","from torch.utils.data import DataLoader\n","import torch.nn as nn\n","import torch.nn.functional as F\n","from sklearn.preprocessing import LabelEncoder, MinMaxScaler\n","from sklearn.model_selection import train_test_split\n","import torch.optim as optim\n","\n","def label_encode_data(df):\n","  df = df.copy(deep=True)\n","  # Encode categorical data\n","  label_encoders = {}\n","  unknown_label = 'unknown'  # Define an unknown label\n","\n","  for column in ['tags', 'title']:\n","      le = LabelEncoder()\n","\n","      # Get unique categories plus an 'unknown' category\n","      unique_categories = df[column].unique().tolist()\n","      # Add 'unknown' to the list of categories\n","      unique_categories.append(unknown_label)\n","\n","      # Fit the LabelEncoder to these categories\n","      le.fit(unique_categories)\n","      df[column] = le.transform(df[column].astype(str))\n","\n","      # Store the encoder\n","      label_encoders[column] = le\n","\n","  return df, label_encoders\n","\n","\n","# Normalize numerical features\n","scaler = MinMaxScaler()\n","df[['listeners', 'playcount']] = scaler.fit_transform(\n","    df[['listeners', 'playcount']])\n","\n","# Label encode categorical features\n","df_scaled, label_encoder_training = label_encode_data(df)\n","\n","# Split data into features and target\n","X = df_scaled[['tags']]\n","y = df_scaled['title']\n","\n","# Split the dataset into training and testing sets\n","X_train, X_test, y_train, y_test = train_test_split(\n","    X, y, test_size=0.2, random_state=42)"]},{"cell_type":"code","execution_count":194,"metadata":{"executionInfo":{"elapsed":166,"status":"ok","timestamp":1715390465207,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"G3RFJN6Ur-s5"},"outputs":[],"source":["class SongRecommender(nn.Module):\n","    def __init__(self):\n","        super(SongRecommender, self).__init__()\n","        self.fc1 = nn.Linear(1, 128)  # Adjust input features if needed\n","        self.fc2 = nn.Linear(128, 256)\n","        self.fc3 = nn.Linear(256, 128)\n","        # Output size = number of unique titles including 'unknown'\n","        # Add 1 for the 'unknown' label\n","        self.output = nn.Linear(128, len(y.unique()) + 1)\n","\n","    def forward(self, x):\n","        x = F.relu(self.fc1(x))\n","        x = F.relu(self.fc2(x))\n","        x = F.relu(self.fc3(x))\n","        x = self.output(x)\n","        return x\n","\n","\n","model = SongRecommender()\n","optimizer = optim.Adam(model.parameters(), lr=0.001)\n","criterion = nn.CrossEntropyLoss()"]},{"cell_type":"code","execution_count":195,"metadata":{"executionInfo":{"elapsed":160,"status":"ok","timestamp":1715390466326,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"-HenNJLnr-s5"},"outputs":[],"source":["def train_model(model, X_train, y_train, X_test, y_test):\n","    train_loader = DataLoader(\n","        list(zip(X_train.values.astype(float), y_train)), batch_size=10, shuffle=True)\n","    test_loader = DataLoader(\n","        list(zip(X_test.values.astype(float), y_test)), batch_size=10, shuffle=False)\n","\n","    model.train()\n","    for epoch in range(10):  # Number of epochs\n","        train_loss = 0\n","        for features, labels in train_loader:\n","            optimizer.zero_grad()\n","            outputs = model(torch.tensor(features).float())\n","            # Ensure labels are long type\n","            loss = criterion(outputs, torch.tensor(labels).long())\n","            loss.backward()\n","            optimizer.step()\n","            train_loss += loss.item()\n","\n","        # Validation phase\n","        model.eval()\n","        validation_loss = 0\n","        for features, labels in test_loader:\n","            outputs = model(torch.tensor(features).float())\n","            loss = criterion(outputs, torch.tensor(labels).long())\n","            validation_loss += loss.item()\n","\n","        print(f'Epoch {epoch+1}, Training Loss: {train_loss / len(train_loader)}, Validation Loss: {validation_loss / len(test_loader)}')"]},{"cell_type":"code","execution_count":196,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":138831,"status":"ok","timestamp":1715390606602,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"gNpxg0ANr-s5","outputId":"c7e9ce0c-3653-4e9a-b3ee-2b9d88da4364"},"outputs":[{"name":"stderr","output_type":"stream","text":["C:\\Users\\Nickk\\AppData\\Local\\Temp\\ipykernel_13264\\1321601871.py:12: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n","  outputs = model(torch.tensor(features).float())\n","C:\\Users\\Nickk\\AppData\\Local\\Temp\\ipykernel_13264\\1321601871.py:14: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n","  loss = criterion(outputs, torch.tensor(labels).long())\n","C:\\Users\\Nickk\\AppData\\Local\\Temp\\ipykernel_13264\\1321601871.py:23: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n","  outputs = model(torch.tensor(features).float())\n","C:\\Users\\Nickk\\AppData\\Local\\Temp\\ipykernel_13264\\1321601871.py:24: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n","  loss = criterion(outputs, torch.tensor(labels).long())\n"]},{"name":"stdout","output_type":"stream","text":["Epoch 1, Training Loss: 14.161600421387472, Validation Loss: 8.646175272324506\n","Epoch 2, Training Loss: 8.468926938374837, Validation Loss: 8.906991397633272\n","Epoch 3, Training Loss: 8.42033219749545, Validation Loss: 9.14518429251278\n","Epoch 4, Training Loss: 8.428513119544512, Validation Loss: 9.366180943507775\n","Epoch 5, Training Loss: 8.350075204872791, Validation Loss: 9.573424189698462\n","Epoch 6, Training Loss: 8.334989405267033, Validation Loss: 9.770331466899199\n","Epoch 7, Training Loss: 8.404972361340935, Validation Loss: 9.958629150016636\n","Epoch 8, Training Loss: 8.490517691624017, Validation Loss: 10.354363404068293\n","Epoch 9, Training Loss: 8.405202573611412, Validation Loss: 10.315738350737329\n","Epoch 10, Training Loss: 8.300552919175889, Validation Loss: 10.487916422825233\n"]}],"source":["train_model(model, X_train, y_train, X_test, y_test)"]},{"cell_type":"code","execution_count":197,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["SongRecommender(\n","  (fc1): Linear(in_features=1, out_features=128, bias=True)\n","  (fc2): Linear(in_features=128, out_features=256, bias=True)\n","  (fc3): Linear(in_features=256, out_features=128, bias=True)\n","  (output): Linear(in_features=128, out_features=4855, bias=True)\n",")\n"]}],"source":["print(model)"]},{"cell_type":"code","execution_count":198,"metadata":{"executionInfo":{"elapsed":138,"status":"ok","timestamp":1715390703802,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"1i30qNdCr-s5"},"outputs":[],"source":["# save the model\n","torch.save(model.state_dict(), './model.pth')"]},{"cell_type":"code","execution_count":199,"metadata":{"executionInfo":{"elapsed":1,"status":"ok","timestamp":1715390703994,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"iCAMAEj5r-s5"},"outputs":[],"source":["# load the model\n","model = SongRecommender()"]},{"cell_type":"code","execution_count":200,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"executionInfo":{"elapsed":292,"status":"ok","timestamp":1715390704465,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"jjkMlHqDHS-Z","outputId":"469d9395-1aa0-4695-98e0-ee867cd31e6b"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>artist_name</th>\n","      <th>title</th>\n","      <th>tags</th>\n","      <th>listeners</th>\n","      <th>playcount</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Sex Pistols</td>\n","      <td>100 Club 1996 ''We Love You Beatles'' - Live</td>\n","      <td>The Beatles, title is a full sentence</td>\n","      <td>0.000070</td>\n","      <td>0.000009</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Baby Rasta &amp; Gringo</td>\n","      <td>Yo Quiero Contigo</td>\n","      <td>Reggaeton, alexis y fido, Eliana, mis videos, ...</td>\n","      <td>0.003978</td>\n","      <td>0.000729</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Bedrock</td>\n","      <td>Emerald</td>\n","      <td>dance</td>\n","      <td>0.000397</td>\n","      <td>0.000097</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>Alicia Keys</td>\n","      <td>Karma</td>\n","      <td>rnb, soul, Alicia Keys, female vocalists, Karma</td>\n","      <td>0.102103</td>\n","      <td>0.044359</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>Joanna Connor</td>\n","      <td>Money Blues</td>\n","      <td>guitar girl, blues</td>\n","      <td>0.000175</td>\n","      <td>0.000043</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["           artist_name                                         title  \\\n","0          Sex Pistols  100 Club 1996 ''We Love You Beatles'' - Live   \n","1  Baby Rasta & Gringo                             Yo Quiero Contigo   \n","4              Bedrock                                       Emerald   \n","6          Alicia Keys                                         Karma   \n","7        Joanna Connor                                   Money Blues   \n","\n","                                                tags  listeners  playcount  \n","0              The Beatles, title is a full sentence   0.000070   0.000009  \n","1  Reggaeton, alexis y fido, Eliana, mis videos, ...   0.003978   0.000729  \n","4                                              dance   0.000397   0.000097  \n","6    rnb, soul, Alicia Keys, female vocalists, Karma   0.102103   0.044359  \n","7                                 guitar girl, blues   0.000175   0.000043  "]},"execution_count":200,"metadata":{},"output_type":"execute_result"}],"source":["df.loc[:, ['artist_name', 'title', 'tags', 'listeners', 'playcount']].head()"]},{"cell_type":"code","execution_count":201,"metadata":{"executionInfo":{"elapsed":186,"status":"ok","timestamp":1715390829249,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"-W3SwScgr-s5"},"outputs":[],"source":["def label_encode_data(df):\n","    df = df.copy(deep=True)\n","    # Encode categorical data\n","    label_encoders = {}\n","    unknown_label = 'unknown'  # Define an unknown label\n","\n","    for column in ['tags']:\n","        le = LabelEncoder()\n","\n","        # Get unique categories plus an 'unknown' category\n","        unique_categories = df[column].unique().tolist()\n","        # Add 'unknown' to the list of categories\n","        unique_categories.append(unknown_label)\n","\n","        # Fit the LabelEncoder to these categories\n","        le.fit(unique_categories)\n","        df[column] = le.transform(df[column].astype(str))\n","\n","        # Store the encoder\n","        label_encoders[column] = le\n","\n","    return df, label_encoders\n","\n","\n","def recommend_songs(model, user_data, full_data=df, train_encoder=label_encoder_training):\n","    model.eval()\n","    full_data = full_data.copy(deep=True)\n","    with torch.no_grad():\n","\n","        # Create a DataFrame with feature names\n","        text_features = user_data.loc[:, ['tags']]\n","\n","        # encoding using concatenated full dataset and evaluation set for inference\n","        df = full_data.loc[:, ['tags']]\n","        text_features_full = df.loc[:, ['tags']]\n","\n","        all_labels = pd.concat([text_features, text_features_full], axis=0)\n","        all_labels.reset_index(drop=True, inplace=True)\n","\n","        # Get the encoder based on all categorical features\n","        _, label_encoders = label_encode_data(all_labels)\n","\n","        # encode the user data\n","        label_encoded_data = text_features.copy(deep=True)\n","        for column in ['tags']:\n","            label_encoded_data[column] = label_encoders[column].transform(\n","                label_encoded_data[column].astype(str))\n","\n","        # converting label_encoded_data into a torch tensor as float dtype\n","        all_features = torch.tensor(\n","            label_encoded_data.to_numpy()).float().unsqueeze(0)\n","\n","        # Make predictions\n","        predictions = model(all_features)\n","\n","        predictions = predictions[0, :5, :]  # selecting top 5\n","        for row in predictions:\n","            top_5_values, top_5_indices = row.topk(5)\n","            recommended_song_ids = top_5_indices.squeeze().tolist()\n","\n","        try:\n","            recommended_titles = label_encoders['title'].inverse_transform(\n","                recommended_song_ids)\n","            recommended_tags = label_encoders['tags'].inverse_transform(\n","                recommended_song_ids)\n","        except:\n","            recommended_titles = train_encoder['title'].inverse_transform(\n","                recommended_song_ids)[:5]\n","            recommended_tags = train_encoder['tags'].inverse_transform(recommended_song_ids)[\n","                :5]\n","\n","        return list(zip(recommended_titles, recommended_tags))"]},{"cell_type":"code","execution_count":202,"metadata":{},"outputs":[],"source":["user_preferences = pd.read_csv(\"..\\\\..\\\\db\\\\data\\\\user_preferences.csv\")\n","user_preferences.drop('level_0', axis=1, inplace=True)"]},{"cell_type":"code","execution_count":203,"metadata":{"executionInfo":{"elapsed":180,"status":"ok","timestamp":1715388203675,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"o89SnI9-r-s5"},"outputs":[],"source":["import requests\n","\n","\n","def fetch_song_data(api_key, artist_name, track_name):\n","    url = \"http://ws.audioscrobbler.com/2.0/\"\n","    params = {\n","        'method': 'track.getInfo',\n","        'api_key': api_key,\n","        'artist': artist_name,\n","        'track': track_name,\n","        'format': 'json'\n","    }\n","    response = requests.get(url, params=params)\n","    print(response.content)\n","    return response.json() if response.status_code == 200 else {}\n","\n","\n","def parse_song_data(song_data):\n","    if song_data and 'track' in song_data:\n","        track = song_data['track']\n","        return {\n","            'artist_name': track['artist']['name'],\n","            'tags': ', '.join([tag['name'] for tag in track.get('toptags', {}).get('tag', [])]),\n","            'duration': float(track.get('duration', 0)),\n","            'listeners': int(track.get('listeners', 0)),\n","            'playcount': int(track.get('playcount', 0)),\n","            'album': track.get('album', {}).get('title', 'Unknown')\n","        }\n","    return {}"]},{"cell_type":"markdown","metadata":{"id":"xm89R7m8Xh-G"},"source":["# Importing the User Data and Making Recommendations\n","Let's make recommendations using the sample user's preferences."]},{"cell_type":"code","execution_count":204,"metadata":{"executionInfo":{"elapsed":171,"status":"ok","timestamp":1715388938035,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"ufzxTjO3YeXT"},"outputs":[],"source":["import numpy as np"]},{"cell_type":"code","execution_count":205,"metadata":{"executionInfo":{"elapsed":379,"status":"ok","timestamp":1715388938616,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"y3hXSZHnXhA7"},"outputs":[],"source":["user_preferences = pd.read_csv(\"..\\\\..\\\\db\\\\data\\\\user_preferences.csv\")\n","user_preferences.drop('level_0', axis=1, inplace=True)"]},{"cell_type":"code","execution_count":206,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":293},"executionInfo":{"elapsed":232,"status":"ok","timestamp":1715388938847,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"OU7aafogGu2t","outputId":"27703100-6719-44ca-8429-cec20e133bbe"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>songID</th>\n","      <th>artist</th>\n","      <th>song</th>\n","      <th>link</th>\n","      <th>text</th>\n","      <th>userID</th>\n","      <th>listeners</th>\n","      <th>playcount</th>\n","      <th>tags</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>19632</td>\n","      <td>Toto</td>\n","      <td>You Are The Flower</td>\n","      <td>/t/toto/you+are+the+flower_20139737.html</td>\n","      <td>You never lose a minute, if in it there is lov...</td>\n","      <td>0</td>\n","      <td>25307</td>\n","      <td>87344</td>\n","      <td>AOR, rock, soft rock, 70s, pop rock</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>19632</td>\n","      <td>Toto</td>\n","      <td>You Are The Flower</td>\n","      <td>/t/toto/you+are+the+flower_20139737.html</td>\n","      <td>You never lose a minute, if in it there is lov...</td>\n","      <td>0</td>\n","      <td>25307</td>\n","      <td>87344</td>\n","      <td>AOR, rock, soft rock, 70s, pop rock</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>25284</td>\n","      <td>Billie Holiday</td>\n","      <td>I Only Have Eyes For You</td>\n","      <td>/b/billie+holiday/i+only+have+eyes+for+you_200...</td>\n","      <td>My love must be a kind of blind love,  \\r\\nI c...</td>\n","      <td>0</td>\n","      <td>60356</td>\n","      <td>178625</td>\n","      <td>jazz, female vocal, vocal jazz, blues, female ...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>43594</td>\n","      <td>Michael Bolton</td>\n","      <td>Only A Woman Like You</td>\n","      <td>/m/michael+bolton/only+a+woman+like+you_101792...</td>\n","      <td>It's beautiful, your honesty  \\r\\nYou cry when...</td>\n","      <td>0</td>\n","      <td>4595</td>\n","      <td>13266</td>\n","      <td>Ballad, romantic, soul, pop, cool</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>50200</td>\n","      <td>Rascal Flatts</td>\n","      <td>The Day Before You</td>\n","      <td>/r/rascal+flatts/the+day+before+you_10238985.html</td>\n","      <td>I had all but given up  \\r\\nOn finding the one...</td>\n","      <td>0</td>\n","      <td>22077</td>\n","      <td>86012</td>\n","      <td>country, rock, contemporary country, seen live...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   songID          artist                      song  \\\n","0   19632            Toto        You Are The Flower   \n","1   19632            Toto        You Are The Flower   \n","2   25284  Billie Holiday  I Only Have Eyes For You   \n","3   43594  Michael Bolton     Only A Woman Like You   \n","4   50200   Rascal Flatts        The Day Before You   \n","\n","                                                link  \\\n","0           /t/toto/you+are+the+flower_20139737.html   \n","1           /t/toto/you+are+the+flower_20139737.html   \n","2  /b/billie+holiday/i+only+have+eyes+for+you_200...   \n","3  /m/michael+bolton/only+a+woman+like+you_101792...   \n","4  /r/rascal+flatts/the+day+before+you_10238985.html   \n","\n","                                                text  userID  listeners  \\\n","0  You never lose a minute, if in it there is lov...       0      25307   \n","1  You never lose a minute, if in it there is lov...       0      25307   \n","2  My love must be a kind of blind love,  \\r\\nI c...       0      60356   \n","3  It's beautiful, your honesty  \\r\\nYou cry when...       0       4595   \n","4  I had all but given up  \\r\\nOn finding the one...       0      22077   \n","\n","   playcount                                               tags  \n","0      87344                AOR, rock, soft rock, 70s, pop rock  \n","1      87344                AOR, rock, soft rock, 70s, pop rock  \n","2     178625  jazz, female vocal, vocal jazz, blues, female ...  \n","3      13266                  Ballad, romantic, soul, pop, cool  \n","4      86012  country, rock, contemporary country, seen live...  "]},"execution_count":206,"metadata":{},"output_type":"execute_result"}],"source":["user_preferences.head()"]},{"cell_type":"code","execution_count":216,"metadata":{"executionInfo":{"elapsed":166,"status":"ok","timestamp":1715388941345,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"jVeX1VX9YXj9"},"outputs":[],"source":["sample_user = user_preferences.where(user_preferences['userID'] == np.random.randint(*(0, 9))).dropna()"]},{"cell_type":"markdown","metadata":{"id":"Yu9C90x4Y9lY"},"source":["Hopefully, the neural network makes recommendations of artists that fall into the top 5 for the user."]},{"cell_type":"code","execution_count":217,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":148,"status":"ok","timestamp":1715391053989,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"wnANm0R3YrVa","outputId":"9f890890-055b-41b9-c630-8986442dcaf9"},"outputs":[{"data":{"text/plain":["tags\n","romantic, Love, pop, easy listening, michael bolton                    5.0\n","pop                                                                    4.0\n","loneliness after dusk, Madonna, demo, never let you go, rebel heart    3.0\n","alternative rock, pop, alternative, pop rock, OneRepublic              3.0\n","pop, boybands, dance, backstreet boys, love at first listen            3.0\n","dtype: float64"]},"execution_count":217,"metadata":{},"output_type":"execute_result"}],"source":["top_5 = sample_user.groupby('tags').count().mean(axis=1).sort_values(ascending=False)[:5]\n","top_5"]},{"cell_type":"code","execution_count":218,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":397,"status":"ok","timestamp":1715390877851,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"m9k_0jVCr-s5","outputId":"9610f359-503f-4b06-d36a-f07d4a79a687"},"outputs":[{"name":"stdout","output_type":"stream","text":["#### RECOMMENDATIONS ###\n"]},{"data":{"text/plain":["[('Blueberry Hill', 'Hip-Hop, hip hop, rap, underground hip-hop, political'),\n"," ('Prognosis', 'mpb, pop, 80s, latin, California'),\n"," ('Money Blues', 'hip hop, rap, Hip-Hop, LL Cool J, Timbaland'),\n"," ('Facedown', 'blues, Old Blues, guitar, slide guitar, gospel'),\n"," ('CB4', 'Kanye West, rnb, 00s, janet jackson, pop')]"]},"execution_count":218,"metadata":{},"output_type":"execute_result"}],"source":["print(\"#### RECOMMENDATIONS ###\")\n","song_recs = recommend_songs(model, sample_user, df) # requires giving main song df for finding embeddings\n","song_recs"]},{"cell_type":"code","execution_count":210,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":276},"executionInfo":{"elapsed":219,"status":"ok","timestamp":1715390907498,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"Klx_gv2v4i6x","outputId":"c2b4dd3e-a48e-411b-9a2c-7963ef922075"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>release</th>\n","      <th>artist_name</th>\n","      <th>duration</th>\n","      <th>artist_familiarity</th>\n","      <th>artist_hotttnesss</th>\n","      <th>year</th>\n","      <th>listeners</th>\n","      <th>playcount</th>\n","      <th>tags</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["Empty DataFrame\n","Columns: [title, release, artist_name, duration, artist_familiarity, artist_hotttnesss, year, listeners, playcount, tags]\n","Index: []"]},"execution_count":210,"metadata":{},"output_type":"execute_result"}],"source":["# finding the song artist in the main dataset\n","df.loc[df['title'].isin(song_recs)]"]},{"cell_type":"code","execution_count":211,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"executionInfo":{"elapsed":428,"status":"ok","timestamp":1715391084128,"user":{"displayName":"Bryan Alexis Ambriz","userId":"16154433038435291108"},"user_tz":420},"id":"AmOx_KyQRALU","outputId":"2010ab1c-ea64-4a00-8dc6-186d584d4868"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>songID</th>\n","      <th>artist</th>\n","      <th>song</th>\n","      <th>link</th>\n","      <th>text</th>\n","      <th>userID</th>\n","      <th>listeners</th>\n","      <th>playcount</th>\n","      <th>tags</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["Empty DataFrame\n","Columns: [songID, artist, song, link, text, userID, listeners, playcount, tags]\n","Index: []"]},"execution_count":211,"metadata":{},"output_type":"execute_result"}],"source":["# lets see how it compares to sample user\n","sample_user.where(sample_user['artist'].isin(top_5.index)).dropna()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"rqkW9oKPRe05"},"outputs":[],"source":[]}],"metadata":{"colab":{"collapsed_sections":["rTCKoervr-s3","9XEtFgNrr-s4"],"provenance":[]},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.9"}},"nbformat":4,"nbformat_minor":0}

mlops/notebooks/scaler.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fec4063e0e914dee18fee94336dd03b1ff78c991d7b22fd4e7aa0c9761bc91f6
+size 1159

mlops/notebooks/test_spotify_api.ipynb

@@ -0,0 +1,368 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view-in-github",
+        "colab_type": "text"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/Bryan-Az/Neurobytes/blob/main/notebooks/test_spotify_api.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 9,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "zQVPVULlQjLy",
+        "outputId": "d949a567-1ee8-4b51-e544-5e6c1888f4e8"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Requirement already satisfied: python-dotenv in /usr/local/lib/python3.10/dist-packages (1.0.1)\n"
+          ]
+        },
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "True"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 9
+        }
+      ],
+      "source": [
+        "import requests\n",
+        "# access colab secrets .env\n",
+        "!pip install python-dotenv\n",
+        "from dotenv import load_dotenv\n",
+        "load_dotenv()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Data Related to the Spotify API"
+      ],
+      "metadata": {
+        "id": "rx--dKrrWFAX"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "Main Discovery Objective:\n",
+        "To identify data that can provide Song-to-Song similarity information based on musicality.\n",
+        "\n",
+        "\n",
+        "---\n",
+        "\n",
+        "The spotify API is able to retrieve data for select artists, albums and shows and other collections within Spotify's content like podcasts or playlists that may help us build alternative sets of information outside of the API from where we can begin."
+      ],
+      "metadata": {
+        "id": "0fZ70UJFWtQr"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import os\n",
+        "SPOTIFY_CLIENT_ID = os.getenv('client_id')\n",
+        "SPOTIFY_CLIENT_SECRET = os.getenv('client_secret')"
+      ],
+      "metadata": {
+        "id": "rj6PkYhtRbex"
+      },
+      "execution_count": 10,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 88,
+      "metadata": {
+        "id": "n8zRio2XQjLz"
+      },
+      "outputs": [],
+      "source": [
+        "def get_access_token(client_id, client_secret):\n",
+        "    url = 'https://accounts.spotify.com/api/token'\n",
+        "    headers = {\n",
+        "        'Content-Type': 'application/x-www-form-urlencoded'\n",
+        "    }\n",
+        "    payload = {\n",
+        "        'grant_type': 'client_credentials'\n",
+        "    }\n",
+        "    response = requests.post(url, headers=headers, data=payload, auth=(client_id, client_secret))\n",
+        "    if response.status_code == 200:\n",
+        "        return response.json()['access_token']\n",
+        "    else:\n",
+        "        raise Exception(\"Failed to retrieve access token\")\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 89,
+      "metadata": {
+        "id": "LGblk66xQjL0"
+      },
+      "outputs": [],
+      "source": [
+        "access_token = get_access_token(SPOTIFY_CLIENT_ID, SPOTIFY_CLIENT_SECRET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Artist-Specific Data"
+      ],
+      "metadata": {
+        "id": "20wfCc7pV-Tk"
+      }
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 95,
+      "metadata": {
+        "id": "KU5bUwqjQjL1"
+      },
+      "outputs": [],
+      "source": [
+        "def get_artist_data(access_token, artist_id):\n",
+        "    '''\n",
+        "    This function retrieves artist data from the Spotify API using the provided access token and artist ID.\n",
+        "    '''\n",
+        "    url = f'https://api.spotify.com/v1/artists/{artist_id}'\n",
+        "    headers = {\n",
+        "        'Authorization': f'Bearer {access_token}'\n",
+        "    }\n",
+        "    response = requests.get(url, headers=headers)\n",
+        "    if response.status_code == 200:\n",
+        "        return response.json()\n",
+        "    else:\n",
+        "        if response.status_code == 404:\n",
+        "            raise Exception(\"Artist not found\")\n",
+        "        elif response.status_code == 401:\n",
+        "            get_access_token(SPOTIFY_CLIENT_ID, SPOTIFY_CLIENT_SECRET)\n",
+        "            return get_artist_data(access_token, artist_id)\n",
+        "        else:\n",
+        "          raise Exception(\"Failed to retrieve artist data\")\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 91,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "DP3CBI7EQjL2",
+        "outputId": "ad4edf96-d819-4bd3-e6e4-09e65895c753"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "{'external_urls': {'spotify': 'https://open.spotify.com/artist/1vCWHaC5f2uS3yhpwWbIA6'}, 'followers': {'href': None, 'total': 22658238}, 'genres': ['dance pop', 'edm', 'pop', 'pop dance'], 'href': 'https://api.spotify.com/v1/artists/1vCWHaC5f2uS3yhpwWbIA6', 'id': '1vCWHaC5f2uS3yhpwWbIA6', 'images': [{'height': 640, 'url': 'https://i.scdn.co/image/ab6761610000e5ebae07171f989fb39736674113', 'width': 640}, {'height': 320, 'url': 'https://i.scdn.co/image/ab67616100005174ae07171f989fb39736674113', 'width': 320}, {'height': 160, 'url': 'https://i.scdn.co/image/ab6761610000f178ae07171f989fb39736674113', 'width': 160}], 'name': 'Avicii', 'popularity': 78, 'type': 'artist', 'uri': 'spotify:artist:1vCWHaC5f2uS3yhpwWbIA6'}\n"
+          ]
+        }
+      ],
+      "source": [
+        "artist_id = '1vCWHaC5f2uS3yhpwWbIA6'  # This is the artist ID for Avicii\n",
+        "artist_data = get_artist_data(access_token, artist_id)\n",
+        "print(artist_data)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "artist_data.keys()"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "lM4V2WtzYrx_",
+        "outputId": "818de95e-ee31-48a0-cbbb-5951f84df08b"
+      },
+      "execution_count": 96,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "dict_keys(['external_urls', 'followers', 'genres', 'href', 'id', 'images', 'name', 'popularity', 'type', 'uri'])"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 96
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "artist_data['genres']"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "ZoG-4QUemGg3",
+        "outputId": "bac86f37-b181-4867-9e02-83d8258aed08"
+      },
+      "execution_count": 127,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "['dance pop', 'edm', 'pop', 'pop dance']"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 127
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "def get_artist_albums(access_token, artist_id):\n",
+        "  '''\n",
+        "  This function retrieves a list of albums for a given artist using the Spotify API.\n",
+        "  returns: list of albums for the artist\n",
+        "  '''\n",
+        "  # part 1 extracts the album information without the tracks\n",
+        "  include_groups = 'album'\n",
+        "  url = f'https://api.spotify.com/v1/artists/{artist_id}/albums/?include_groups={include_groups}'\n",
+        "  headers = {\n",
+        "      'Authorization': f'Bearer {access_token}'\n",
+        "  }\n",
+        "  response = requests.get(url, headers=headers)\n",
+        "  if response.status_code == 200:\n",
+        "        return response.json()\n",
+        "  else:\n",
+        "      if response.status_code == 404:\n",
+        "          raise Exception(\"Artist not found\")\n",
+        "      elif response.status_code == 401:\n",
+        "          get_access_token(SPOTIFY_CLIENT_ID, SPOTIFY_CLIENT_SECRET)\n",
+        "          return 'Try again, new token generated'\n",
+        "      else:\n",
+        "        raise Exception(\"Failed to retrieve artist data\")"
+      ],
+      "metadata": {
+        "id": "lRMDVG0CY_xr"
+      },
+      "execution_count": 122,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# setting include_group to album only to return only this artists own albums\n",
+        "artist_albums = get_artist_albums(access_token, artist_id)"
+      ],
+      "metadata": {
+        "id": "IXZFbWG-ZGvN"
+      },
+      "execution_count": 123,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "The available metadata available for the artist album's returned."
+      ],
+      "metadata": {
+        "id": "KzKBzVnBa2Gc"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# might be redundant information if the include_group is set\n",
+        "for i in artist_albums:\n",
+        "  print(i['name'] + ' ' + i['album_group'] + ' '+ i['album_type'])"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "lfEf-p2ybHWW",
+        "outputId": "fe39c159-cefa-4633-83a6-798e0ce1a651"
+      },
+      "execution_count": 78,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "TIM album album\n",
+            "Stories album album\n",
+            "True: Avicii By Avicii album album\n",
+            "The Days / Nights album album\n",
+            "True (Bonus Edition) album album\n",
+            "True album album\n",
+            "Malo (The Cube Guys Remix) single single\n",
+            "Street Dancer (Sgt Slick's Discotizer 2022 Remix) single single\n",
+            "My Feelings For You (Mark Knight Remix) single single\n",
+            "My Feelings For You (Don Diablo Remix) single single\n",
+            "Forever Yours (Avicii Tribute) single single\n",
+            "Fades Away (feat. MishCatt) [Tribute Concert Version] single single\n",
+            "Heaven (David Guetta & MORTEN Remix) single single\n",
+            "Tough Love (Tiësto Remix) single single\n",
+            "SOS (Laidback Luke Tribute Remix) single single\n",
+            "Tough Love (feat. Vargas & Lagola) single single\n",
+            "SOS (feat. Aloe Blacc) single single\n",
+            "Lonely Together (Remixes) single single\n",
+            "Lonely Together (Acoustic) single single\n",
+            "Without You (Remixes) single single\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [],
+      "metadata": {
+        "id": "G4IPmxegYmHN"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.8.19"
+    },
+    "colab": {
+      "provenance": [],
+      "include_colab_link": true
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

mlops/scripts/create_user_preferences/requirements.txt

@@ -0,0 +1,11 @@
+pandas==1.3.3
+nltk
+scikit-learn
+pydrive
+oauth2client
+google-cloud-storage
+google-auth
+google-auth-oauthlib
+tqdm
+python-dotenv
+os

mlops/scripts/create_user_preferences/run.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Step a: Install dependencies
+python3 -m venv venv
+source venv/bin/activate
+pip install -r requirements.txt
+
+# Step b: Run necessary parts of the codebase
+#python src/document_term_matrix.py &
+#python src/cosine_similarity.py &
+python src/synthetic_user_data.py &
+
+wait

mlops/scripts/create_user_preferences/run_search.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Step a: Install dependencies
+python3 -m venv venv
+source venv/bin/activate
+pip install -r requirements.txt
+
+python3 src/data_loader.py &
+
+wait

mlops/scripts/create_user_preferences/src/config.py

@@ -0,0 +1,6 @@
+import os
+
+# Configuration for data paths
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+DATA_DIR = os.path.dirname(os.path.abspath('../../../db/data/README.md'))
+PROJECT_DIR = os.path.dirname(os.path.abspath('../../../README.md'))

mlops/scripts/create_user_preferences/src/cosine_similarity.py

@@ -0,0 +1,13 @@
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
+from document_term_matrix import create_document_term_matrix
+from data_loader import load_data
+from config import DATA_DIR
+def calculate_cosine_similarity(dtm):
+    """ Calculate the cosine similarity matrix from a document-term matrix. """
+    return cosine_similarity(dtm)
+
+if __name__ == "__main__":
+    data = load_data('millionsong_dataset.zip')
+    dtm, features = create_document_term_matrix(data['text'])
+    similarity_matrix = calculate_cosine_similarity(dtm)

mlops/scripts/create_user_preferences/src/data_loader.py

@@ -0,0 +1,154 @@
+import pandas as pd
+from config import DATA_DIR, PROJECT_DIR
+import os
+import io
+import google
+from google.oauth2 import credentials
+from google.oauth2.credentials import Credentials
+from google_auth_oauthlib.flow import InstalledAppFlow
+from googleapiclient.discovery import build
+from googleapiclient.http import MediaIoBaseDownload
+from googleapiclient.http import MediaFileUpload
+import socket
+
+def is_port_in_use(port):
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        return s.connect_ex(('localhost', port)) == 0
+
+def load_credentials():
+    """Load or create new credentials."""
+    SCOPES = ['https://www.googleapis.com/auth/drive']
+    creds = None
+    # The file token.json stores the user's access and refresh tokens, and is
+    # created automatically when the authorization flow completes for the first
+    # time.
+    try:
+        creds, _ = google.auth.default()
+    except google.auth.exceptions.DefaultCredentialsError:
+        # Run the flow using the client secrets file
+        if not is_port_in_use(8080):
+            path_to_json = PROJECT_DIR + "/client_secrets.json"  # Path relative to your main application file
+            print(path_to_json)
+            flow = InstalledAppFlow.from_client_secrets_file(path_to_json, SCOPES)
+            creds = flow.run_local_server(port=8080, access_type='offline', prompt='consent')
+            with open('token.json', 'w') as token:
+                token.write(creds.to_json())
+            creds = credentials.Credentials.from_authorized_user_file('token.json', SCOPES)
+            if isinstance(creds, Credentials):
+                print("creds is a valid Credentials object.")
+            else:
+                print("creds is not a valid Credentials object.")
+        else:
+            print('In else in load creds')
+            # Load the credentials from token.json if they exist and check for expiry
+            try:
+                with open('token.json', 'r') as token:
+                    creds = credentials.Credentials.from_authorized_user_file('token.json', SCOPES)
+                if isinstance(creds, Credentials):
+                    print("creds is a valid Credentials object.")
+                else:
+                    print("creds is not a valid Credentials object.")
+            except FileNotFoundError:
+                print("Token file not found. Please re-run the authentication flow.")
+                raise
+    return creds
+
+def search_files(service, file_name):
+    '''
+    helps search for files in the neurobytes google folder
+    '''
+    items = []
+    try:
+        creds = load_credentials()
+        print('credentials loaded')
+        # Link PyDrive to use the credentials
+        service = build('drive', 'v3', credentials=creds)
+        print('auth with gdrive')
+
+        results = service.files().list(
+            pageSize=10,
+            fields="nextPageToken, files(id, name)",
+            q=f"name='{file_name}' and trashed=false"
+        ).execute()
+        items = results.get('files', [])
+
+        if not items:
+            print('No files found.')
+        else:
+            print('Files:')
+            for item in items:
+                print(u'{0} ({1})'.format(item['name'], item['id']))
+
+    except Exception as e:
+        print('An error occured in auth with google')
+        print(e)
+    return items
+
+def load_data(file_id, file_name, n=10000):
+    """
+    Load data from a specified file within the data directory.
+    Only a sample of 2000 rows will be used to load data
+    as the original dataset is very large.
+    Args:
+    file_name (str): The name of the file to load.
+    
+    Returns:
+    DataFrame: A pandas DataFrame containing the loaded data.
+    """
+    if os.path.exists(DATA_DIR + '/' + file_name):
+        file_path = os.path.join(DATA_DIR, file_name)
+        data = pd.read_csv(file_path)
+        print(f"Data loaded successfully from {file_path}")
+        sample_data = data.sample(n, replace=False).reset_index(drop=False)
+        return sample_data
+    else:
+        try:
+            creds = load_credentials()
+            print('credentials loaded')
+            # Link PyDrive to use the credentials
+            service = build('drive', 'v3', credentials=creds)
+            print('auth with gdrive')
+            request = service.files().get_media(fileId=file_id)
+            fh = io.BytesIO()
+            downloader = MediaIoBaseDownload(fh, request, chunksize=204800)  # Adjust chunk size as needed
+            done = False
+            try:
+                while not done:
+                    status, done = downloader.next_chunk()
+                    print("Download progress: {0}".format(status.progress() * 100))
+                fh.seek(0)
+                with open(DATA_DIR + '/'+ file_name, 'wb') as f:
+                    f.write(fh.read())
+                print(f"Download of '{DATA_DIR + '/'+ file_name}' complete.")
+            except Exception as e:
+                print(f"An error occurred: {e}")
+
+                print('Data accessed from glink')
+            file_path = os.path.join(DATA_DIR, file_name)
+            data = pd.read_csv(file_path)
+            print(f"Data loaded successfully from {file_path}")
+            sample_data = data.sample(n, replace=False).reset_index(drop=False)
+            return sample_data
+
+        except Exception as e:
+            print('An error occured in auth with google')
+            print(e)
+
+def upload_data(filename, creds):
+    """Uploads a file to Google Drive."""
+    SCOPES = ['https://www.googleapis.com/auth/drive.file']
+    service = build('drive', 'v3', credentials=creds)
+    neurobytes_folder = '1VxknqmOtEsoCM3R0DQEOYpjMVUOHW24H'
+    file_metadata = {
+        'name': os.path.basename(filename),
+        'parents': [neurobytes_folder]
+    }
+    media = MediaFileUpload(filename, mimetype='text/csv')
+    file = service.files().create(body=file_metadata, media_body=media, fields='id').execute()
+    print(f"File ID: {file.get('id')}")
+
+if __name__ == '__main__':
+    creds = load_credentials()
+    service = build('drive', 'v3', credentials=creds)
+    file_name = "CMPE-258: Team Neurobytes"
+    search_files(service, file_name)

mlops/scripts/create_user_preferences/src/document_term_matrix.py

@@ -0,0 +1,45 @@
+import pandas as pd
+from sklearn.feature_extraction.text import CountVectorizer
+from config import DATA_DIR
+from data_loader import load_data
+
+def create_document_term_matrix(data):
+    """ Convert text data into a document-term matrix. """
+    text_corpus = create_text_corpus(data)
+    vectorizer = CountVectorizer()
+    dtm = vectorizer.fit_transform(text_corpus)
+    return dtm, vectorizer.get_feature_names_out()
+
+from nltk.tokenize import word_tokenize
+from nltk.probability import FreqDist
+import nltk
+nltk.download('punkt')
+nltk.download('stopwords')
+def create_text_corpus(data, lyric_chars=1000):
+    """
+    Prepares the text data before input to the document-term matrix.
+    It incorporates artist name and a subset of lyrics.
+    """
+
+    def top_n_words(text, n=5):
+        '''
+        Applied on the lyrics to shrink weight of lyrics on similarity
+        '''
+        freq_dist = FreqDist(word_tokenize(text))
+        # remove filler words
+        freq_dist = FreqDist({key: val for key, val in freq_dist.items() if key not in nltk.corpus.stopwords.words('english')})
+        # select only the words from the (word, freq) tuple list
+        words = ' '.join([word for word, freq in freq_dist.most_common()])
+        return words
+    
+    artist_names = data['artist']
+    song_name = data['song']
+    lyrics = data['text']
+    top_words = lyrics.apply(lambda x: top_n_words(x)) 
+    
+    return artist_names + ' ' + song_name + ' ' + top_words
+
+
+if __name__ == "__main__":
+    data = load_data('millionsong_dataset.zip')
+    dtm, features = create_document_term_matrix(data)

mlops/scripts/create_user_preferences/src/integrate_lastfm.py

@@ -0,0 +1,82 @@
+from dotenv import load_dotenv
+# load LASTFM_API_KEY from .env
+import requests
+import os
+
+def fetch_data(api_key, method, params):
+    base_url = "http://ws.audioscrobbler.com/2.0/"
+    params['api_key'] = api_key
+    params['method'] = method
+    params['format'] = 'json'
+    response = requests.get(base_url, params=params)
+    return response.json()
+
+
+def get_artist_info(api_key, artist_name):
+    params = {'artist': artist_name}
+    return fetch_data(api_key, 'artist.getInfo', params)
+
+
+def get_track_info(api_key, artist_name, track_name):
+    params = {'artist': artist_name, 'track': track_name}
+    return fetch_data(api_key, 'track.getInfo', params)
+
+
+def batch_fetch_data(api_key, items, fetch_function, sleep_time=1):
+    results = []
+    for item in items:
+        result = fetch_function(api_key, *item)
+        results.append(result)
+        # time.sleep(sleep_time)
+    return results
+
+api_key = os.getenv('LASTFM_API_KEY')
+
+
+def fetch_lastfm_data(api_key, artist_name, track_name):
+    base_url = "http://ws.audioscrobbler.com/2.0/"
+    params = {
+        'method': 'track.getInfo',
+        'api_key': api_key,
+        'artist': artist_name,
+        'track': track_name,
+        'format': 'json'
+    }
+    response = requests.get(base_url, params=params)
+    if response.status_code == 200 and response.text.strip():
+        return response.json()
+    else:
+        return None
+
+
+def parse_lastfm_data(data):
+    if data and 'track' in data:
+        track = data['track']
+        return {
+            'listeners': track.get('listeners', '0'),
+            'playcount': track.get('playcount', '0'),
+            'tags': ', '.join(tag['name'] for tag in track.get('toptags', {}).get('tag', [])),
+        }
+    return None
+
+from tqdm import tqdm
+tqdm.pandas()
+
+load_dotenv()
+api_key = os.getenv('LASTFM_API_KEY')
+tracks_skipped = 0
+
+def print_tracks_skipped():
+    print(f"Tracks skipped: {tracks_skipped}")
+
+
+def fetch_and_parse(row):
+    global tracks_skipped
+    data = fetch_lastfm_data(api_key, row['artist'], row['song'])
+    if data is None:
+        tracks_skipped += 1
+        return None
+    parsed_data = parse_lastfm_data(data)
+    if parsed_data is None:
+        tracks_skipped += 1
+    return parsed_data

mlops/scripts/create_user_preferences/src/synthetic_user_data.py

@@ -0,0 +1,108 @@
+import numpy as np
+from config import DATA_DIR
+import pandas as pd
+import os
+from data_loader import load_credentials, load_data, upload_data
+from cosine_similarity import create_document_term_matrix, calculate_cosine_similarity
+from integrate_lastfm import *
+
+creds = load_credentials()
+
+def save_similarity_matrix(matrix, feature_names, filepath):
+    matrix = pd.DataFrame(matrix, index=feature_names, columns=feature_names)
+    matrix.to_csv(filepath)
+    return matrix
+
+def generate_user_preferences(similarity_matrix, feature_names, num_users=10, songs_per_user=(50,100), top_similar=(5, 10)):
+    """ Generate synthetic user data based on artist similarity. """
+    sim_df = save_similarity_matrix(similarity_matrix, feature_names, DATA_DIR + '/song_similarity_matrix.csv')
+    song_index = similarity_matrix.shape[0]
+    # dictionary assigning masks of indices-to-song in the similarity matrix to users
+    # requires selecting the data from the similarity matrix using the masks for semantics
+    user_data = {}
+    top_similar_n = np.random.randint(*top_similar)
+    for user_id in range(num_users):
+        num_songs = np.random.randint(*songs_per_user)
+        # samples 40-50 starter songs
+        starter_songs = np.random.choice(song_index, size=num_songs, replace=False)
+        
+        # selects these indices from the pandas df
+        user_sim_df = sim_df.iloc[starter_songs,:]
+        def select_top_similar(row, n=top_similar_n):
+            '''
+            selects top n similar songs to the starter song and adds it to user preferences
+            '''
+
+            curr_song_index = row.name # include starter song
+            row = row.to_numpy()
+            # excluding songs with same lyrics as starter
+            non_starters = row != 1.0
+            row = row[non_starters]
+            top_n_indices = np.array([])
+            if len(row) >= n:
+                # selecting top 3 similarities
+                top_n_indices_mask = np.argpartition(row, -n)[-n-1:-1]
+                # sorting the top 3 similarities
+                top_n_indices_mask = top_n_indices_mask[np.argsort(row[top_n_indices_mask])[::-1]]
+                # returning the indices 
+                top_n_indices = np.where(non_starters)[0][top_n_indices_mask]
+            # adding original starter song to preferences
+            top_n_indices = np.insert(top_n_indices, 0, curr_song_index)
+            if user_id not in user_data:
+                user_data[user_id] = top_n_indices.tolist()
+            else:
+                user_data[user_id].extend(top_n_indices.tolist())
+        user_sim_df.apply(lambda row: select_top_similar(row), axis=1)
+
+    return user_data
+
+def transform_mask_to_songs(row_index, array, data):
+    curr_user_index = row_index
+    row = np.array(array[1])
+    user_songs = data.iloc[row, :].copy(deep=True)
+    user_songs['userID'] = curr_user_index
+
+    return user_songs
+
+def save_preferences(user_preferences):
+    '''
+    saves data locally and then uploads to google drive
+    '''
+    filename = DATA_DIR + '/user_preferences.csv'
+    user_preferences.reset_index(drop=False, inplace=True)
+    user_preferences.rename(columns={'index':'songID'}, inplace=True)
+    user_preferences.to_csv(filename, index=False)
+    print('Synthetic User Data Saved to Local File: ' + DATA_DIR + '/user_preferences.csv')
+    upload_data(filename, creds=creds)
+    print('Synthetic User Data Saved to Remote Drive.')
+
+def integrate_lastfm(user_preferences):
+  ### Adding additional LastFM Data for inference with NN Model ###
+    user_preferences['lastfm_data'] =  user_preferences.progress_apply(fetch_and_parse, axis=1)
+    user_preferences = user_preferences[user_preferences['lastfm_data'].notna()]
+    user_preferences.reset_index(drop=True, inplace=True)
+    track_details_df = pd.json_normalize(user_preferences['lastfm_data'])
+    mixed = pd.concat(
+        [user_preferences.drop(columns=['lastfm_data']), track_details_df], axis=1)
+    print_tracks_skipped()
+    return mixed
+
+
+if __name__ == "__main__":
+    file_id = '1EL4vYhO4A0Cgm8akBgAfDrWOGvtF6Xvo'
+    file_name = 'millionsong_dataset.zip'
+    data = load_data(file_id=file_id, file_name=file_name)
+
+    ### Creating the Synthetic user preferences
+    dtm, lyric_term_features = create_document_term_matrix(data)
+    song_artist_index = list(data.index)
+    similarity_matrix = calculate_cosine_similarity(dtm)
+    user_preference_masks = generate_user_preferences(similarity_matrix, feature_names=song_artist_index)
+    user_preferences = pd.concat([transform_mask_to_songs(row, array, data) for row, array in enumerate(user_preference_masks.items())], axis = 0)
+
+    ### Connecting user data to additional LastFM features for model inference
+    user_preferences = integrate_lastfm(user_preferences)
+    save_preferences(user_preferences=user_preferences)
+    
+    
+

mlops/scripts/train_model/train_model.py

@@ -0,0 +1,190 @@
+!pip install python-dotenv
+# imports
+import pandas as pd
+import h5py
+import os
+from sqlalchemy import create_engine
+import requests
+import time
+from dotenv import load_dotenv
+import pandas as pd
+
+df = pd.read_csv('/content/drive/MyDrive/CMPE-258: Team Neurobytes/Neurobytes/db/data/music_data.csv')
+df.dropna(inplace=True)
+
+import pandas as pd
+import torch
+from torch.utils.data import DataLoader
+import torch.nn as nn
+import torch.nn.functional as F
+from sklearn.preprocessing import LabelEncoder, MinMaxScaler
+from sklearn.model_selection import train_test_split
+import torch.optim as optim
+
+# Encode categorical data
+label_encoders = {}
+unknown_label = 'unknown'  # Define an unknown label
+
+for column in ['artist_name', 'tags', 'title']:
+    le = LabelEncoder()
+
+    # Get unique categories plus an 'unknown' category
+    unique_categories = df[column].unique().tolist()
+    # Add 'unknown' to the list of categories
+    unique_categories.append(unknown_label)
+
+    # Fit the LabelEncoder to these categories
+    le.fit(unique_categories)
+    df[column] = le.transform(df[column].astype(str))
+
+    # Store the encoder
+    label_encoders[column] = le
+
+
+# Normalize numerical features
+scaler = MinMaxScaler()
+df[['duration', 'listeners', 'playcount']] = scaler.fit_transform(
+    df[['duration', 'listeners', 'playcount']])
+
+# Split data into features and target
+X = df[['artist_name', 'tags', 'duration', 'listeners', 'playcount']]
+y = df['title']
+
+# Split the dataset into training and testing sets
+X_train, X_test, y_train, y_test = train_test_split(
+    X, y, test_size=0.2, random_state=42)
+
+class SongRecommender(nn.Module):
+  def __init__(self):
+      super(SongRecommender, self).__init__()
+      self.fc1 = nn.Linear(5, 128)  # Adjust input features if needed
+      self.fc2 = nn.Linear(128, 256)
+      self.fc3 = nn.Linear(256, 128)
+      # Output size = number of unique titles including 'unknown'
+      # Add 1 for the 'unknown' label
+      self.output = nn.Linear(128, len(y.unique()) + 1)
+
+  def forward(self, x):
+      x = F.relu(self.fc1(x))
+      x = F.relu(self.fc2(x))
+      x = F.relu(self.fc3(x))
+      x = self.output(x)
+      return x
+
+
+model = SongRecommender()
+optimizer = optim.Adam(model.parameters(), lr=0.001)
+criterion = nn.CrossEntropyLoss()
+
+def train_model(model, X_train, y_train, X_test, y_test):
+    train_loader = DataLoader(
+        list(zip(X_train.values.astype(float), y_train)), batch_size=50, shuffle=True)
+    test_loader = DataLoader(
+        list(zip(X_test.values.astype(float), y_test)), batch_size=50, shuffle=False)
+
+    model.train()
+    for epoch in range(50):  # Number of epochs
+        train_loss = 0
+        for features, labels in train_loader:
+            optimizer.zero_grad()
+            outputs = model(torch.tensor(features).float())
+            # Ensure labels are long type
+            loss = criterion(outputs, torch.tensor(labels).long())
+            loss.backward()
+            optimizer.step()
+            train_loss += loss.item()
+
+        # Validation phase
+        model.eval()
+        validation_loss = 0
+        for features, labels in test_loader:
+            outputs = model(torch.tensor(features).float())
+            loss = criterion(outputs, torch.tensor(labels).long())
+            validation_loss += loss.item()
+
+        print(f'Epoch {epoch+1}, Training Loss: {train_loss / len(train_loader)}, Validation Loss: {validation_loss / len(test_loader)}')
+train_model(model, X_train, y_train, X_test, y_test)
+# save the model
+torch.save(model.state_dict(), 'model.pth')
+# load the model
+model = SongRecommender()
+def recommend_songs(model, input_features):
+    model.eval()
+    print(input_features)
+    with torch.no_grad():
+        try:
+            artist_index = label_encoders['artist_name'].transform(
+                [input_features['artist_name']])
+        except ValueError:
+            artist_index = label_encoders['artist_name'].transform(['unknown'])
+
+        try:
+            tags_index = label_encoders['tags'].transform(
+                [input_features['tags']])
+        except ValueError:
+            tags_index = label_encoders['tags'].transform(['unknown'])
+
+        # Create a DataFrame with feature names
+        scaled_features = pd.DataFrame(
+            [[input_features['duration'], input_features['listeners'],
+                input_features['playcount']]],
+            columns=['duration', 'listeners', 'playcount']
+        )
+        scaled_features = scaler.transform(scaled_features)[0]
+
+        features = torch.tensor(
+            [artist_index[0], tags_index[0], *scaled_features]).float().unsqueeze(0)
+        predictions = model(features)
+        top_5_values, top_5_indices = predictions.topk(5)
+        recommended_song_ids = top_5_indices.squeeze().tolist()
+        
+        return label_encoders['title'].inverse_transform(recommended_song_ids)
+
+import requests
+
+def fetch_song_data(api_key, artist_name, track_name):
+    url = "http://ws.audioscrobbler.com/2.0/"
+    params = {
+        'method': 'track.getInfo',
+        'api_key': api_key,
+        'artist': artist_name,
+        'track': track_name,
+        'format': 'json'
+    }
+    response = requests.get(url, params=params)
+    print(response.content)
+    return response.json() if response.status_code == 200 else {}
+
+
+def parse_song_data(song_data):
+    if song_data and 'track' in song_data:
+        track = song_data['track']
+        return {
+            'artist_name': track['artist']['name'],
+            'tags': ', '.join([tag['name'] for tag in track.get('toptags', {}).get('tag', [])]),
+            'duration': float(track.get('duration', 0)),
+            'listeners': int(track.get('listeners', 0)),
+            'playcount': int(track.get('playcount', 0)),
+            'album': track.get('album', {}).get('title', 'Unknown')
+        }
+    return {}
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+api_key = os.getenv('LASTFM_API_KEY')
+
+artist_name = 'Lagy Gaga'
+track_name = 'Poker Face'
+
+# Fetch and parse song data
+song_data = fetch_song_data(api_key, artist_name, track_name)
+parsed_data = parse_song_data(song_data)
+
+print(song_data)
+# if the song is not found, or the tags column is empty, print a message
+if not parsed_data or not parsed_data['tags']:
+    print("Song not found or tags not available.")
+
+else:
+    recommend_songs(model, parsed_data)

src/model_evaluation_v2.ipynb

@@ -0,0 +1,136 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# **Evaluating the Recommendation Model**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/mocha/miniconda3/envs/mamba/envs/neurobytes_music_recommender/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import gradio as gr\n",
+    "import torch\n",
+    "import torch.nn as nn\n",
+    "from joblib import load\n",
+    "import sklearn"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Define the same neural network model\n",
+    "class ImprovedSongRecommender(nn.Module):\n",
+    "    def __init__(self, input_size, num_titles):\n",
+    "        super(ImprovedSongRecommender, self).__init__()\n",
+    "        self.fc1 = nn.Linear(input_size, 128)\n",
+    "        self.bn1 = nn.BatchNorm1d(128)\n",
+    "        self.fc2 = nn.Linear(128, 256)\n",
+    "        self.bn2 = nn.BatchNorm1d(256)\n",
+    "        self.fc3 = nn.Linear(256, 128)\n",
+    "        self.bn3 = nn.BatchNorm1d(128)\n",
+    "        self.output = nn.Linear(128, num_titles)\n",
+    "        self.dropout = nn.Dropout(0.5)\n",
+    "\n",
+    "    def forward(self, x):\n",
+    "        x = torch.relu(self.bn1(self.fc1(x)))\n",
+    "        x = self.dropout(x)\n",
+    "        x = torch.relu(self.bn2(self.fc2(x)))\n",
+    "        x = self.dropout(x)\n",
+    "        x = torch.relu(self.bn3(self.fc3(x)))\n",
+    "        x = self.dropout(x)\n",
+    "        x = self.output(x)\n",
+    "        return x\n",
+    "\n",
+    "# Load the trained model\n",
+    "model_path = \"../models/improved_model.pth\"\n",
+    "num_unique_titles = 4855  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = ImprovedSongRecommender(input_size=2, num_titles=num_unique_titles)  \n",
+    "model.load_state_dict(torch.load(model_path, map_location=torch.device('cpu')))\n",
+    "model.eval()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/mocha/miniconda3/envs/mamba/envs/neurobytes_music_recommender/lib/python3.8/site-packages/sklearn/base.py:348: InconsistentVersionWarning: Trying to unpickle estimator LabelEncoder from version 1.2.2 when using version 1.3.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
+      "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n",
+      "  warnings.warn(\n",
+      "/Users/mocha/miniconda3/envs/mamba/envs/neurobytes_music_recommender/lib/python3.8/site-packages/sklearn/base.py:348: InconsistentVersionWarning: Trying to unpickle estimator MinMaxScaler from version 1.2.2 when using version 1.3.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
+      "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Load the label encoders and scaler\n",
+    "label_encoders_path = \"data/new_label_encoders.joblib\"\n",
+    "scaler_path = \"data/new_scaler.joblib\"\n",
+    "\n",
+    "label_encoders = load(label_encoders_path)\n",
+    "scaler = load(scaler_path)\n",
+    "\n",
+    "# Create a mapping from encoded indices to actual song titles\n",
+    "index_to_song_title = {index: title for index, title in enumerate(label_encoders['title'].classes_)}\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/model_training_v2.ipynb

@@ -1,36 +1,22 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "provenance": []
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "language_info": {
-      "name": "python"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
-      "source": [
-        "# **Music recommender**"
-      ],
       "metadata": {
         "id": "DDADPl-phDUC"
-      }
+      },
+      "source": [
+        "# **Music recommender**"
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "# **Load Data**"
-      ],
       "metadata": {
         "id": "E7Cu5Fmqct7J"
-      }
+      },
+      "source": [
+        "# **Load Data**"
+      ]
     },
     {
       "cell_type": "code",
@@ -45,11 +31,7 @@
       },
       "outputs": [
         {
-          "output_type": "display_data",
           "data": {
-            "text/plain": [
-              "<IPython.core.display.HTML object>"
-            ],
             "text/html": [
               "\n",
               "     <input type=\"file\" id=\"files-793c32c8-99a6-4873-9585-738e1d4b2ab1\" name=\"files[]\" multiple disabled\n",
@@ -234,13 +216,17 @@
               "};\n",
               "})(self);\n",
               "</script> "
+            ],
+            "text/plain": [
+              "<IPython.core.display.HTML object>"
             ]
           },
-          "metadata": {}
+          "metadata": {},
+          "output_type": "display_data"
         },
         {
-          "output_type": "stream",
           "name": "stdout",
+          "output_type": "stream",
           "text": [
             "Saving music_data.csv to music_data.csv\n",
             "                                          title  \\\n",
@@ -293,9 +279,7 @@
     },
     {
       "cell_type": "code",
-      "source": [
-        "df.head()"
-      ],
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/",
@@ -304,40 +288,14 @@
         "id": "9E3in0U3dK5I",
         "outputId": "c1d5362a-6a33-4543-ff4d-4e11cf8220ec"
       },
-      "execution_count": null,
       "outputs": [
         {
-          "output_type": "execute_result",
           "data": {
-            "text/plain": [
-              "                                          title  \\\n",
-              "0  100 Club 1996 ''We Love You Beatles'' - Live   \n",
-              "1                             Yo Quiero Contigo   \n",
-              "4                                       Emerald   \n",
-              "6                                         Karma   \n",
-              "7                                   Money Blues   \n",
-              "\n",
-              "                           release          artist_name   duration  \\\n",
-              "0     Sex Pistols - The Interviews          Sex Pistols   88.73751   \n",
-              "1  Sentenciados - Platinum Edition  Baby Rasta & Gringo  167.36608   \n",
-              "4                          Emerald              Bedrock  501.86404   \n",
-              "6         The Diary Of Alicia Keys          Alicia Keys  255.99955   \n",
-              "7                        Slidetime        Joanna Connor  243.66975   \n",
-              "\n",
-              "   artist_familiarity  artist_hotttnesss  year  listeners  playcount  \\\n",
-              "0            0.731184           0.549204     0        172        210   \n",
-              "1            0.610186           0.355320     0       9753      16911   \n",
-              "4            0.654039           0.390625  2004        973       2247   \n",
-              "6            0.933916           0.778674  2003     250304    1028356   \n",
-              "7            0.479218           0.332857     0        429       1008   \n",
-              "\n",
-              "                                                tags  \n",
-              "0              The Beatles, title is a full sentence  \n",
-              "1  Reggaeton, alexis y fido, Eliana, mis videos, ...  \n",
-              "4                                              dance  \n",
-              "6    rnb, soul, Alicia Keys, female vocalists, Karma  \n",
-              "7                                 guitar girl, blues  "
-            ],
+            "application/vnd.google.colaboratory.intrinsic+json": {
+              "summary": "{\n  \"name\": \"df\",\n  \"rows\": 5063,\n  \"fields\": [\n    {\n      \"column\": \"title\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4854,\n        \"samples\": [\n          \"I Wish I Had A Girl\",\n          \"Jump [Jacques Lu Cont Edit]\",\n          \"Mulin' Around\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"release\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4187,\n        \"samples\": [\n          \"Le Bordel Magnifique\",\n          \"Charlotte's Web (OST)\",\n          \"X.O. Experience\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"artist_name\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2461,\n        \"samples\": [\n          \"Lee Ritenour\",\n          \"Pennywise\",\n          \"Anneli Drecker\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"duration\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 107.73289375974717,\n        \"min\": 1.04444,\n        \"max\": 1815.2224,\n        \"num_unique_values\": 3939,\n        \"samples\": [\n          294.24281,\n          240.79628,\n          115.53914\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"artist_familiarity\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.14886096792686204,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 2474,\n        \"samples\": [\n          0.787098355481,\n          0.481771820142,\n          0.374024633035\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"artist_hotttnesss\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.1347303774485448,\n        \"min\": 0.0,\n        \"max\": 1.08250255673,\n        \"num_unique_values\": 2398,\n        \"samples\": [\n          0.376018761952,\n          0.355667956383,\n          0.289970666912\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"year\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 917,\n        \"min\": 0,\n        \"max\": 2010,\n        \"num_unique_values\": 69,\n        \"samples\": [\n          1979,\n          0,\n          1965\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"listeners\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 150513,\n        \"min\": 0,\n        \"max\": 2451482,\n        \"num_unique_values\": 3914,\n        \"samples\": [\n          781546,\n          6216,\n          396579\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"playcount\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1115103,\n        \"min\": 0,\n        \"max\": 23182516,\n        \"num_unique_values\": 4422,\n        \"samples\": [\n          62736,\n          1305,\n          17033\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"tags\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4583,\n        \"samples\": [\n          \"dance, 90s, trance, House, jungle\",\n          \"country, favorite songs, classic country, linedance, Martina McBride\",\n          \"90s, heavy metal, thrash metal, metal, punk\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}",
+              "type": "dataframe",
+              "variable_name": "df"
+            },
             "text/html": [
               "\n",
               "  <div id=\"df-b9e5c35d-1534-4ad7-8661-887b39a472e9\" class=\"colab-df-container\">\n",
@@ -650,31 +608,48 @@
               "    </div>\n",
               "  </div>\n"
             ],
-            "application/vnd.google.colaboratory.intrinsic+json": {
-              "type": "dataframe",
-              "variable_name": "df",
-              "summary": "{\n  \"name\": \"df\",\n  \"rows\": 5063,\n  \"fields\": [\n    {\n      \"column\": \"title\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4854,\n        \"samples\": [\n          \"I Wish I Had A Girl\",\n          \"Jump [Jacques Lu Cont Edit]\",\n          \"Mulin' Around\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"release\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4187,\n        \"samples\": [\n          \"Le Bordel Magnifique\",\n          \"Charlotte's Web (OST)\",\n          \"X.O. Experience\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"artist_name\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2461,\n        \"samples\": [\n          \"Lee Ritenour\",\n          \"Pennywise\",\n          \"Anneli Drecker\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"duration\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 107.73289375974717,\n        \"min\": 1.04444,\n        \"max\": 1815.2224,\n        \"num_unique_values\": 3939,\n        \"samples\": [\n          294.24281,\n          240.79628,\n          115.53914\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"artist_familiarity\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.14886096792686204,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 2474,\n        \"samples\": [\n          0.787098355481,\n          0.481771820142,\n          0.374024633035\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"artist_hotttnesss\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.1347303774485448,\n        \"min\": 0.0,\n        \"max\": 1.08250255673,\n        \"num_unique_values\": 2398,\n        \"samples\": [\n          0.376018761952,\n          0.355667956383,\n          0.289970666912\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"year\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 917,\n        \"min\": 0,\n        \"max\": 2010,\n        \"num_unique_values\": 69,\n        \"samples\": [\n          1979,\n          0,\n          1965\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"listeners\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 150513,\n        \"min\": 0,\n        \"max\": 2451482,\n        \"num_unique_values\": 3914,\n        \"samples\": [\n          781546,\n          6216,\n          396579\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"playcount\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1115103,\n        \"min\": 0,\n        \"max\": 23182516,\n        \"num_unique_values\": 4422,\n        \"samples\": [\n          62736,\n          1305,\n          17033\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"tags\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4583,\n        \"samples\": [\n          \"dance, 90s, trance, House, jungle\",\n          \"country, favorite songs, classic country, linedance, Martina McBride\",\n          \"90s, heavy metal, thrash metal, metal, punk\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
-            }
+            "text/plain": [
+              "                                          title  \\\n",
+              "0  100 Club 1996 ''We Love You Beatles'' - Live   \n",
+              "1                             Yo Quiero Contigo   \n",
+              "4                                       Emerald   \n",
+              "6                                         Karma   \n",
+              "7                                   Money Blues   \n",
+              "\n",
+              "                           release          artist_name   duration  \\\n",
+              "0     Sex Pistols - The Interviews          Sex Pistols   88.73751   \n",
+              "1  Sentenciados - Platinum Edition  Baby Rasta & Gringo  167.36608   \n",
+              "4                          Emerald              Bedrock  501.86404   \n",
+              "6         The Diary Of Alicia Keys          Alicia Keys  255.99955   \n",
+              "7                        Slidetime        Joanna Connor  243.66975   \n",
+              "\n",
+              "   artist_familiarity  artist_hotttnesss  year  listeners  playcount  \\\n",
+              "0            0.731184           0.549204     0        172        210   \n",
+              "1            0.610186           0.355320     0       9753      16911   \n",
+              "4            0.654039           0.390625  2004        973       2247   \n",
+              "6            0.933916           0.778674  2003     250304    1028356   \n",
+              "7            0.479218           0.332857     0        429       1008   \n",
+              "\n",
+              "                                                tags  \n",
+              "0              The Beatles, title is a full sentence  \n",
+              "1  Reggaeton, alexis y fido, Eliana, mis videos, ...  \n",
+              "4                                              dance  \n",
+              "6    rnb, soul, Alicia Keys, female vocalists, Karma  \n",
+              "7                                 guitar girl, blues  "
+            ]
           },
+          "execution_count": 2,
           "metadata": {},
-          "execution_count": 2
+          "output_type": "execute_result"
         }
+      ],
+      "source": [
+        "df.head()"
       ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "# Display basic information about the dataset\n",
-        "print(df.info())\n",
-        "\n",
-        "# Display summary statistics for numerical columns\n",
-        "print(df.describe())\n",
-        "\n",
-        "# Display unique values for categorical columns\n",
-        "print(\"Unique values in 'title':\", df['title'].nunique())\n",
-        "print(\"Unique values in 'artist_name':\", df['artist_name'].nunique())\n",
-        "print(\"Unique values in 'tags':\", df['tags'].nunique())"
-      ],
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -682,11 +657,10 @@
         "id": "b_sSacbdHcn6",
         "outputId": "f745b028-fd97-4b19-b9f0-9e041621e5d3"
       },
-      "execution_count": null,
       "outputs": [
         {
-          "output_type": "stream",
           "name": "stdout",
+          "output_type": "stream",
           "text": [
             "<class 'pandas.core.frame.DataFrame'>\n",
             "Index: 5063 entries, 0 to 9530\n",
@@ -730,19 +704,48 @@
             "Unique values in 'tags': 4583\n"
           ]
         }
+      ],
+      "source": [
+        "# Display basic information about the dataset\n",
+        "print(df.info())\n",
+        "\n",
+        "# Display summary statistics for numerical columns\n",
+        "print(df.describe())\n",
+        "\n",
+        "# Display unique values for categorical columns\n",
+        "print(\"Unique values in 'title':\", df['title'].nunique())\n",
+        "print(\"Unique values in 'artist_name':\", df['artist_name'].nunique())\n",
+        "print(\"Unique values in 'tags':\", df['tags'].nunique())"
       ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "# **Preprocessing**"
-      ],
       "metadata": {
         "id": "wPVFDtk9g9ox"
-      }
+      },
+      "source": [
+        "# **Preprocessing**"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "3fsU1IvylyZg",
+        "outputId": "c2ba3adc-c077-454a-94de-ca9bb0ba4807"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Label encoders and scaler saved successfully.\n"
+          ]
+        }
+      ],
       "source": [
         "import pandas as pd\n",
         "from sklearn.preprocessing import LabelEncoder, MinMaxScaler\n",
@@ -790,27 +793,32 @@
         "joblib.dump(scaler, \"/content/new_scaler.joblib\")\n",
         "\n",
         "print(\"Label encoders and scaler saved successfully.\")\n"
-      ],
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
         },
-        "id": "3fsU1IvylyZg",
-        "outputId": "c2ba3adc-c077-454a-94de-ca9bb0ba4807"
+        "id": "JBWZWp_8Jr82",
+        "outputId": "73a312c1-3615-4a87-965b-c2fc41fc50e7"
       },
-      "execution_count": null,
       "outputs": [
         {
-          "output_type": "stream",
           "name": "stdout",
+          "output_type": "stream",
           "text": [
-            "Label encoders and scaler saved successfully.\n"
+            "Data split into training and testing sets.\n",
+            "Maximum value in y_train: 4854\n",
+            "Maximum value in y_test: 4850\n",
+            "Number of unique titles: 4855\n",
+            "Maximum value in y_train after clipping: 4854\n",
+            "Maximum value in y_test after clipping: 4850\n"
           ]
         }
-      ]
-    },
-    {
-      "cell_type": "code",
+      ],
       "source": [
         "from sklearn.model_selection import train_test_split\n",
         "\n",
@@ -846,41 +854,41 @@
         "# Print the maximum values after clipping\n",
         "print(\"Maximum value in y_train after clipping:\", y_train.max())\n",
         "print(\"Maximum value in y_test after clipping:\", y_test.max())\n"
-      ],
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "syYhdUbxgA-K"
+      },
+      "source": [
+        "# **Training**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
         },
-        "id": "JBWZWp_8Jr82",
-        "outputId": "73a312c1-3615-4a87-965b-c2fc41fc50e7"
+        "id": "aaR1IGymKQq2",
+        "outputId": "9e5115a5-1a75-4672-a0b3-4fdd314e1a79"
       },
-      "execution_count": null,
       "outputs": [
         {
-          "output_type": "stream",
           "name": "stdout",
+          "output_type": "stream",
           "text": [
-            "Data split into training and testing sets.\n",
-            "Maximum value in y_train: 4854\n",
-            "Maximum value in y_test: 4850\n",
-            "Number of unique titles: 4855\n",
-            "Maximum value in y_train after clipping: 4854\n",
-            "Maximum value in y_test after clipping: 4850\n"
+            "Epoch 1, Training Loss: 8.921830113728841, Validation Loss: 8.836441385979747\n",
+            "Epoch 2, Training Loss: 8.331391870239635, Validation Loss: 9.148561271966672\n",
+            "Epoch 3, Training Loss: 7.494005516429007, Validation Loss: 10.484928570541681\n",
+            "Epoch 4, Training Loss: 6.704833826606657, Validation Loss: 11.745069999320835\n",
+            "Early stopping triggered\n",
+            "Improved model trained and saved successfully.\n"
           ]
         }
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "# **Training**"
       ],
-      "metadata": {
-        "id": "syYhdUbxgA-K"
-      }
-    },
-    {
-      "cell_type": "code",
       "source": [
         "import torch\n",
         "import torch.nn as nn\n",
@@ -981,41 +989,36 @@
         "torch.save(model.state_dict(), model_path)\n",
         "\n",
         "print(\"Improved model trained and saved successfully.\")\n"
-      ],
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "g4hJVlNXf5Vu"
+      },
+      "source": [
+        "# **Testing**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
         },
-        "id": "aaR1IGymKQq2",
-        "outputId": "9e5115a5-1a75-4672-a0b3-4fdd314e1a79"
+        "id": "KwqV-HnCOvtz",
+        "outputId": "d412ce92-3ab8-4f3d-df83-22ef9e857203"
       },
-      "execution_count": null,
       "outputs": [
         {
-          "output_type": "stream",
           "name": "stdout",
+          "output_type": "stream",
           "text": [
-            "Epoch 1, Training Loss: 8.921830113728841, Validation Loss: 8.836441385979747\n",
-            "Epoch 2, Training Loss: 8.331391870239635, Validation Loss: 9.148561271966672\n",
-            "Epoch 3, Training Loss: 7.494005516429007, Validation Loss: 10.484928570541681\n",
-            "Epoch 4, Training Loss: 6.704833826606657, Validation Loss: 11.745069999320835\n",
-            "Early stopping triggered\n",
-            "Improved model trained and saved successfully.\n"
+            "Recommendations: ['Betrayal Is A Symptom', 'The Earth Will Shake', 'Saturday', 'Firehouse Rock', 'Breathe Easy']\n"
           ]
         }
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "# **Testing**"
       ],
-      "metadata": {
-        "id": "g4hJVlNXf5Vu"
-      }
-    },
-    {
-      "cell_type": "code",
       "source": [
         "import torch\n",
         "from joblib import load\n",
@@ -1095,27 +1098,28 @@
         "\n",
         "recommendations = recommend_songs(tags, artist_name)\n",
         "print(\"Recommendations:\", recommendations)\n"
-      ],
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
         },
-        "id": "KwqV-HnCOvtz",
-        "outputId": "d412ce92-3ab8-4f3d-df83-22ef9e857203"
+        "id": "3HzLKv5mPxOv",
+        "outputId": "62b37d04-4857-44fb-b5c4-8ead55db9b1a"
       },
-      "execution_count": null,
       "outputs": [
         {
-          "output_type": "stream",
           "name": "stdout",
+          "output_type": "stream",
           "text": [
-            "Recommendations: ['Betrayal Is A Symptom', 'The Earth Will Shake', 'Saturday', 'Firehouse Rock', 'Breathe Easy']\n"
+            "Recommendations: ['Betrayal Is A Symptom', 'Carnival (from \"Black Orpheus\")', 'Saturday', 'The Earth Will Shake', 'Start!']\n",
+            "Recommendations: ['Old Friends', 'Betrayal Is A Symptom', 'Between Love & Hate', 'Carnival (from \"Black Orpheus\")', 'Satin Doll']\n"
           ]
         }
-      ]
-    },
-    {
-      "cell_type": "code",
+      ],
       "source": [
         "import torch\n",
         "from joblib import load\n",
@@ -1202,25 +1206,22 @@
         "\n",
         "recommendations = recommend_songs(tags, artist_name)\n",
         "print(\"Recommendations:\", recommendations)\n"
-      ],
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "id": "3HzLKv5mPxOv",
-        "outputId": "62b37d04-4857-44fb-b5c4-8ead55db9b1a"
-      },
-      "execution_count": null,
-      "outputs": [
-        {
-          "output_type": "stream",
-          "name": "stdout",
-          "text": [
-            "Recommendations: ['Betrayal Is A Symptom', 'Carnival (from \"Black Orpheus\")', 'Saturday', 'The Earth Will Shake', 'Start!']\n",
-            "Recommendations: ['Old Friends', 'Betrayal Is A Symptom', 'Between Love & Hate', 'Carnival (from \"Black Orpheus\")', 'Satin Doll']\n"
-          ]
-        }
       ]
     }
-  ]
-}
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python",
+      "version": "3.8.1"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}