<jupyter_start><jupyter_text>Load UMAP model and analysis dataframe<jupyter_code>working_dir = '/data/datasets/organoid_phenotyping/analysis/arlotta_vs_lancaster/'
os.listdir(working_dir)
umap = joblib.load(os.path.join(working_dir, 'model_arlotta.umap'))
analysis = pd.read_csv(os.path.join(working_dir, 'analysis.csv'), index_col=0)
analysis.head()<jupyter_output><empty_output><jupyter_text>Get all profiles and labels<jupyter_code>n = 1000

np.random.seed(1)
dfs = []
for org in tqdm(analysis.index, total=len(analysis)):
    folder = analysis['type'].loc[org]
    org_dir = os.path.join(working_dir, folder, org)
    profiles = np.load(os.path.join(org_dir, 'dataset/cyto_profiles_sample.npy'))
    labels = np.load(os.path.join(org_dir, 'cyto_labels.npy'))
    profiles_sample, labels_sample = randomly_sample(n, profiles, labels)
    x = umap.transform(profiles_sample.reshape((len(profiles_sample), -1)))
    df = pd.DataFrame({'x': x[:, 0], 
                       'y': x[:, 1],
                       'label': labels_sample,
                       'organoid': len(x) * [org], 
                       'type': len(x) * [folder]})
    dfs.append(df)
df = pd.concat(dfs)
df.head()

plt.figure(figsize=(6, 8))
for i in range(6):
    df_org = df.iloc[i*n:(i+1)*n]
    x = df_org['x']
    y = df_org['y']
    plt.subplot(3, 2, i+1)
    sns.scatterplot(x=x, y=y, edgecolor=None, s=5)
    plt.xlim([-10, 10])
    plt.ylim([-10, 10])
plt.tight_layout()
plt.show()
plt.figure(figsize=(6, 12))
for i in range(8):
    df_org = df.iloc[i*n+6*n:(i+1)*n+6*n]
    x = df_org['x']
    y = df_org['y']
    plt.subplot(4, 2, i+1)
    sns.scatterplot(x=x, y=y, edgecolor=None, s=5)
    plt.xlim([-10, 10])
    plt.ylim([-10, 10])
plt.tight_layout()
plt.show()
lancaster_orgs = [1, 2, 5]
arlotta_orgs = [2, 3, 4]

plt.figure(figsize=(10, 6))
for i, org in enumerate(lancaster_orgs):
    start = org * n + 6*n
    stop = (org+1) * n + 6*n
    
    df_org = df.iloc[start:stop]
    x = df_org['x']
    y = df_org['y']
    
    plt.subplot(2, 3, i+1)
    sns.scatterplot(x=x, y=y, edgecolor=None, s=5)
    plt.title(f'Lancaster, {i+1}')
    plt.ylim([-10, 10])
    plt.xlim([-12, 8])
    
for i, org in enumerate(arlotta_orgs):
    start = org * n
    stop = (org+1) * n
    
    df_org = df.iloc[start:stop]
    x = df_org['x']
    y = df_org['y']
    
    plt.subplot(2, 3, i+1+3)
    sns.scatterplot(x=x, y=y, edgecolor=None, s=5)
    plt.title(f'Arlotta, {i+1}')
    plt.ylim([-10, 10])
    plt.xlim([-12, 8])
    
sns.despine()
plt.tight_layout()
plt.show()
lancaster_orgs = [1, 2, 5]
arlotta_orgs = [2, 3, 4]

plt.figure(figsize=(4, 8))
for i, org in enumerate(lancaster_orgs):
    start = org * n + 6*n
    stop = (org+1) * n + 6*n
    
    df_org = df.iloc[start:stop]
    x = df_org['x']
    y = df_org['y']
    
    plt.subplot(2, 1, 1)
    sns.scatterplot(x=x, y=y, edgecolor=None, s=5, alpha=0.5)
    plt.title(f'Lancaster')
    plt.ylim([-10, 10])
    plt.xlim([-12, 8])
    
for i, org in enumerate(arlotta_orgs):
    start = org * n
    stop = (org+1) * n
    
    df_org = df.iloc[start:stop]
    x = df_org['x']
    y = df_org['y']
    
    plt.subplot(2, 1, 2)
    sns.scatterplot(x=x, y=y, edgecolor=None, s=5, alpha=0.5)
    plt.title(f'Arlotta')
    plt.ylim([-10, 10])
    plt.xlim([-12, 8])
    
sns.despine()
plt.tight_layout()
plt.savefig(os.path.join(working_dir, 'umap_arlotta_consistency.pdf'), bbox_inches='tight')
plt.show()
n_orgs = 4

plt.figure(figsize=(6, 18))

plt.subplot(3, 1, 1)
sns.scatterplot(x='x', y='y', hue='type', data=df, edgecolor=None, s=2, alpha=0.3, palette=['b', 'r'])
plt.title('Arlotta vs Lancaster')
plt.xlabel('UMAP 1')
plt.ylabel('UMAP 2')

plt.subplot(3, 1, 2)
sns.scatterplot(x='x', y='y', hue='organoid', data=df.where(df['type'] == 'Arlotta_d56').dropna().iloc[1*n:(n_orgs+1)*n], 
                edgecolor=None, s=3, alpha=1, legend=None)
plt.title('Arlotta d56')
plt.xlabel('UMAP 1')
plt.ylabel('UMAP 2')

plt.subplot(3, 1, 3)
sns.scatterplot(x='x', y='y', hue='organoid', data=df.where(df['type'] == 'Lancaster_d60').dropna().iloc[2*n:(n_orgs+2)*n], 
                edgecolor=None, s=3, alpha=1, legend=None)
plt.title('Lancaster d60')
plt.xlabel('UMAP 1')
plt.ylabel('UMAP 2')

plt.tight_layout()
sns.despine()
plt.savefig(os.path.join(working_dir, 'umap_arlotta_vs_lancaster.pdf'), bbox_inches='tight')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19868001820664544, 'fraction_numerical': 0.035958124715521164, 'mean_word_length': 3.5076923076923077, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '595494', 'license': 'MIT', 'n_tokens_mistral': 2082, 'n_tokens_neox': 1936, 'n_words': 335, 'path': 'notebooks/Figure 7 - UMAP Arlotta vs Lancaster cyto  consistency.ipynb', 'repo_name': 'renardeinside/scout', 'size': '4394'}