<jupyter_start><jupyter_text>Load UMAP model and analysis dataframe<jupyter_code>working_dir = '/data/datasets/organoid_phenotyping/analysis/d35_vs_d60/'
umap = joblib.load(os.path.join(working_dir, 'model_d35_d60.umap'))
analysis = pd.read_csv(os.path.join(working_dir, 'analysis.csv'), index_col=0)
analysis<jupyter_output><empty_output><jupyter_text>Get all profiles and labels<jupyter_code>n = 5000

np.random.seed(1)
dfs = []
for org in tqdm(analysis.index, total=len(analysis)):
    folder = analysis['type'].loc[org]
    org_dir = os.path.join(working_dir, folder, org)
    profiles = np.load(os.path.join(org_dir, 'dataset/cyto_profiles.npy'))
#     sample_idx = np.load(os.path.join(org_dir, 'dataset/cyto_sample_index.npy'))
    labels = np.load(os.path.join(org_dir, 'cyto_labels.npy'))
    
    profiles_sample, labels_sample = randomly_sample(n, profiles, labels)
    x = umap.transform(profiles_sample.reshape((len(profiles_sample), -1)))
    df = pd.DataFrame({'x': x[:, 0], 
                       'y': x[:, 1],
                       'label': labels_sample,
                       'organoid': len(x) * [org], 
                       'type': len(x) * [folder]})
    dfs.append(df)
df = pd.concat(dfs)
df['type'].unique()
n_orgs = 4

plt.figure(figsize=(6, 18))

plt.subplot(3, 1, 1)
sns.scatterplot(x='x', y='y', hue='type', data=df, edgecolor=None, s=2, alpha=0.3, palette=['b', 'r'])
plt.title('d35 vs d60')
plt.xlabel('UMAP 1')
plt.ylabel('UMAP 2')

plt.subplot(3, 1, 2)
sns.scatterplot(x='x', y='y', hue='organoid', data=df.where(df['type'] == 'Lancaster_d35').dropna().iloc[:n_orgs*n], 
                edgecolor=None, s=3, alpha=1, legend=None)
plt.title('d35')
plt.xlabel('UMAP 1')
plt.ylabel('UMAP 2')

plt.subplot(3, 1, 3)
sns.scatterplot(x='x', y='y', hue='organoid', data=df.where(df['type'] == 'Lancaster_d60').dropna().iloc[2*n:(n_orgs+2)*n], 
                edgecolor=None, s=3, alpha=1, legend=None)
plt.title('d60')
plt.xlabel('UMAP 1')
plt.ylabel('UMAP 2')

plt.tight_layout()
sns.despine()
plt.savefig(os.path.join(working_dir, 'umap_d35_vs_d60.pdf'), bbox_inches='tight')
plt.show()
plt.figure(figsize=(6, 6))

sns.scatterplot(x='x', y='y', hue='label', data=df)

plt.tight_layout()
sns.despine()
# plt.savefig(os.path.join(working_dir, 'umap_d35_vs_d60_clusters.pdf'), bbox_inches='tight')
plt.show()
df.where(df['label'] == 0).dropna()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19683728672492717, 'fraction_numerical': 0.028297960882230546, 'mean_word_length': 4.616822429906542, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '525229', 'license': 'MIT', 'n_tokens_mistral': 1057, 'n_tokens_neox': 980, 'n_words': 146, 'path': 'notebooks/Figure 5 - UMAP d60 vs d35.ipynb', 'repo_name': 'renardeinside/scout', 'size': '2403'}