<jupyter_start><jupyter_text>load data<jupyter_code>DATASET_ID = 'BIRD_DB_Vireo_cassinii'
df_loc =  DATA_DIR / 'syllable_dfs' / DATASET_ID / 'cassins.pickle'
syllable_df = pd.read_pickle(df_loc)
del syllable_df['audio']
syllable_df[:3]
len(syllable_df)
top_indv_df = pd.DataFrame(
    {indv: [np.sum(syllable_df.indv == indv)] for indv in syllable_df.indv.unique()}
).T.sort_values(by=[0], ascending=False)
top_indv_df[:15]
top_indvs = top_indv_df[:15].index
top_indvs
np.shape(syllable_df.spectrogram.values[0])<jupyter_output><empty_output><jupyter_text>project<jupyter_code>for indv in tqdm(top_indvs):
    subset_df = syllable_df[syllable_df.indv == indv]
    
    specs = list(subset_df.spectrogram.values)
    specs = [i/np.max(i) for i in tqdm(specs, leave=False)]
    specs_flattened = flatten_spectrograms(specs)
    print(np.shape(specs_flattened))
    
    cuml_umap = cumlUMAP()
    embedding = cuml_umap.fit_transform(specs_flattened)
    subset_df['umap'] = list(embedding)
    
    unique_labs = np.unique(subset_df.labels.values)
    unique_labs_dict = {lab:i for i, lab in enumerate(unique_labs)}
    lab_list = [unique_labs_dict[i] for i in subset_df.labels.values]
    fig, ax = plt.subplots()
    ax.scatter(embedding[:,0], embedding[:,1], s=1, c=lab_list, cmap=plt.cm.tab20, alpha = 0.25)
    #ax.set_xlim([-8,8])
    #ax.set_ylim([-8,8])
    plt.show()
    
    ensure_dir(DATA_DIR / 'embeddings' / DATASET_ID / 'indvs')
    subset_df.to_pickle(DATA_DIR / 'embeddings' / DATASET_ID / 'indvs' / (indv + '.pickle'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14440894568690096, 'fraction_numerical': 0.012140575079872205, 'mean_word_length': 4.887218045112782, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '217836', 'license': 'MIT', 'n_tokens_mistral': 687, 'n_tokens_neox': 681, 'n_words': 92, 'path': 'notebooks/02.5-make-projection-dfs/multiple_spread/cassins-umap-indv.ipynb', 'repo_name': 'xingjeffrey/avgn_paper', 'size': '1565'}