<jupyter_start><jupyter_text>Extract unique track ids spotify:track:5OSaZxhNj307YpTh7Qp8Xy<jupyter_code>import json
import numpy as np
from time import time
import pandas as pd
from pandas.io.json import json_normalize
import os
import random
import gc
gc.collect()<jupyter_output><empty_output><jupyter_text>Reading track_uri<jupyter_code>path = '../dataset/spotify_million_playlist_dataset/data/'
t0 = time()
samp = 1
lst = []

filenames = os.listdir(path)

for i, filename in enumerate(sorted(filenames)):
    if filename.startswith("mpd.slice.") and filename.endswith(".json"):
        fullpath = os.sep.join((path, filename))
        f = open(fullpath)
        js = f.read()
        f.close()
        mpd_slice = json.loads(js)
        for playlist in mpd_slice['playlists']:
            for track in playlist['tracks']:
                lst.append([track['track_uri'],playlist['pid']])
    print(filename, i)

    # Time diff
print(f"Time taken: {(time()-t0)/60}")
len(lst)
lst_uri = [el[0] for el in lst if el[0]]
set_uri = set(lst_uri)
len(set_uri)
track_uri = np.array(set_uri)
pd.DataFrame(set_uri).to_csv('../data/track_uris.csv', index=None)
check = pd.read_csv('../data/track_uris.csv')
check.head()
check.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14046288906624102, 'fraction_numerical': 0.010375099760574621, 'mean_word_length': 4.26890756302521, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '562156', 'license': 'MIT', 'n_tokens_mistral': 473, 'n_tokens_neox': 461, 'n_words': 93, 'path': 'data-processing-notebooks/track-uri-combine.ipynb', 'repo_name': 'IrynaHorova/APC-Spotify-Million-Playlists-Ironhack-Project', 'size': '1253'}