<jupyter_start><jupyter_text>load data<jupyter_code>DATASET_ID = 'swamp_sparrow'
df_loc =  DATA_DIR / 'syllable_dfs' / DATASET_ID / 'swampsparrow-taper_with_freq.pickle'
syllable_df = pd.read_pickle(df_loc)
del syllable_df['audio']
syllable_df[:3]
fig, ax = plt.subplots()
plt.hist(syllable_df.pos_in_syllable.values, bins=np.arange(10));
ax.set_xlim([0,10])
top_indv_df = pd.DataFrame(
    {indv: [np.sum(syllable_df.indv == indv)] for indv in syllable_df.indv.unique()}
).T.sort_values(by=[0], ascending=False)
top_indv_df.iloc[100]
top_indvs = top_indv_df.index[:100]
top_indvs<jupyter_output><empty_output><jupyter_text>project<jupyter_code>for indv in tqdm(top_indvs):
    subset_df = syllable_df[syllable_df.indv == indv]
    if len(subset_df) < 100:
        continue
    
    print(indv)
    specs = list(subset_df.spectrogram.values)
    specs = [i / np.max(i) for i in tqdm(specs, leave=False)]
    specs_flattened = flatten_spectrograms(specs)
    print(np.shape(specs_flattened))

    cuml_umap = cumlUMAP()
    embedding = cuml_umap.fit_transform(specs_flattened)
    subset_df["umap"] = list(embedding)

    unique_labs = np.unique(subset_df.pos_in_syllable.values)
    unique_labs_dict = {lab: i for i, lab in enumerate(unique_labs)}
    lab_list = [unique_labs_dict[i] for i in subset_df.pos_in_syllable.values]
    fig, ax = plt.subplots()
    ax.scatter(
        embedding[:, 0], embedding[:, 1], s=2, c=lab_list, cmap=plt.cm.tab20, alpha=1
    )
    #ax.set_xlim([-15, 15])
    #ax.set_ylim([-15, 15])
    plt.show()

    ensure_dir(DATA_DIR / 'embeddings' / DATASET_ID / 'indvs')
    subset_df.to_pickle(DATA_DIR / 'embeddings' / DATASET_ID / 'indvs' / (indv + '.pickle'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14235909355026147, 'fraction_numerical': 0.017431725740848343, 'mean_word_length': 4.6644736842105265, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '394657', 'license': 'MIT', 'n_tokens_mistral': 762, 'n_tokens_neox': 740, 'n_words': 107, 'path': 'notebooks/02.5-make-projection-dfs/swamp-sparrow-umap-indv.ipynb', 'repo_name': 'xingjeffrey/avgn_paper', 'size': '1721'}