<jupyter_start><jupyter_text>Read playlists<jupyter_code>t0 = time()
keys = [
    'pid',
    'name',
    'description',
    'num_artists',
    'num_albums',
    'num_tracks',
    'num_followers',
    'duration_ms',
    'collaborative',
    'tracks'
]
samp = 700
arr = np.empty(shape=(samp*1000,len(keys)), dtype = object)
path='spotify_million_playlist_dataset/data/'
filenames = os.listdir(path)
for i, filename in enumerate(random.sample(sorted(filenames), samp)):
    if filename.startswith("mpd.slice.") and filename.endswith(".json"):
        fullpath = os.sep.join((path, filename))
        f = open(fullpath)
        js = f.read()
        f.close()
        mpd_slice = json.loads(js)
        D = pd.DataFrame(mpd_slice['playlists'])[keys].to_numpy()
        arr[i*1000:(i+1)*1000,:]= D
        print(filename,i)
# Time diff
print(f"Time taken: {(time()-t0)/60}")
arr[0]
arr[1]<jupyter_output><empty_output><jupyter_text>Reading track_uri<jupyter_code>lst = []
for playlist in data['playlists'][:2]:
    for track in playlist['tracks']:
        lst.append([track['track_uri'],playlist['pid']])
t0 = time()
samp = 100
lst = []

path='spotify_million_playlist_dataset/data/'
filenames = os.listdir(path)

for i, filename in enumerate(sorted(filenames)):
    if filename.startswith("mpd.slice.") and filename.endswith(".json"):
        fullpath = os.sep.join((path, filename))
        f = open(fullpath)
        js = f.read()
        f.close()
        mpd_slice = json.loads(js)
        for playlist in mpd_slice['playlists']:
            for track in playlist['tracks']:
                lst.append([track['track_uri'],playlist['pid']])
    print(filename, i)
# Time diff
print(f"Time taken: {(time()-t0)/60}")
len(lst)
lst_uri = [el[0] for el in lst if el[0]]
set_uri = set(lst_uri)
len(set_uri)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1694915254237288, 'fraction_numerical': 0.017495899398578457, 'mean_word_length': 3.6923076923076925, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '428586', 'license': 'MIT', 'n_tokens_mistral': 702, 'n_tokens_neox': 674, 'n_words': 120, 'path': 'data-processing-notebooks/.ipynb_checkpoints/track-uri-combine-checkpoint.ipynb', 'repo_name': 'IrynaHorova/APC-Spotify-Million-Playlists-Ironhack-Project', 'size': '1829'}