<jupyter_start><jupyter_text>Projection ('PCA densities') of (scGen) simulated LD-SPL cells  on the combined 10 donors (base normalised) dataset reference --- hpb29 Date: 2021-02-09<jupyter_code>%matplotlib widget

import warnings
warnings.filterwarnings('ignore')

import os, sys, json, operator, getpass
from pathlib import Path
from datetime import datetime

import numpy as np
import pandas as pd
import scanpy as sc

import matplotlib.pyplot as plt
from ipywidgets import widgets

from scipy.spatial import distance
from collections import Counter

from scipy import stats
home = str(Path.home())
user = getpass.getuser()

basedir = os.path.join(home, 'datafloor/users', user, '2020/SLX19841/')

sc.settings.writedir = os.path.join(basedir, 'analysis/h5ad/')
sc.settings.verbosity = 3
sc.settings.file_format_figs = 'svg'
sc.settings.savefigs = False
sc.set_figure_params(dpi=150)<jupyter_output><empty_output><jupyter_text>--- Read reference dataset<jupyter_code>data = sc.read('COMBO10_NO_SPL3_Seurat3_lognorm')<jupyter_output><empty_output><jupyter_text>Read predicted dataset (to project on reference)<jupyter_code>prediction_100e = sc.read('COMBO10_SPL_lognorm_LD_prediction_hvg')<jupyter_output><empty_output><jupyter_text>transfer annotations from dod SPL cells to predicted LD SPL cells<jupyter_code>prediction_100e.obs = data[data.obs.organ == 'SPL', :].obs.copy()<jupyter_output><empty_output><jupyter_text>re-adjust indexes in order to distinguish from source SPL cells<jupyter_code>prediction_100e.obs.index = prediction_100e.obs.index.copy()+'.p'<jupyter_output><empty_output><jupyter_text>annotate condition (measured vs predicted)<jupyter_code>prediction_100e.obs['condition'] = 'pred_SPL'<jupyter_output><empty_output><jupyter_text>'PCA density' projection ---<jupyter_code>data.obs['condition'] = 'reference'<jupyter_output><empty_output><jupyter_text>Concatenate base normalised dataset reference w/ LD-SPL prediction dataset<jupyter_code>%%time
concat_data = data.concatenate(prediction_100e)
%%time
sc.pp.scale(concat_data)
%%time
sc.pp.pca(concat_data, svd_solver='arpack')
%%time
X_pca = concat_data.obsm['X_pca']

D = distance.squareform(distance.pdist(X_pca, metric='euclidean'))<jupyter_output>CPU times: user 7min 29s, sys: 1min 14s, total: 8min 43s
Wall time: 8min 43s<jupyter_text>---  Projection densities<jupyter_code>distances = {}

distances['pred'] = D[len(data.obs_names):len(data.obs_names)+ len(prediction_100e.obs_names), ]
distances['pred'] = distances['pred'].T
distances['pred'] = distances['pred'][:len(data.obs_names),]
distances['pred'] = distances['pred'].T
distances['pred'].shape
counts = {}

top_ones = {}
most_freq = {}
    
# initialize reference counts matrix with zeros
counts['pred'] = np.zeros(len(data.obs_names))

# 'proportionate' number of neighbors considered...
k = int(100000/len(prediction_100e.obs_names))
k
%%time
# for each of my sample's cells...
for j in range(len(prediction_100e.obs_names)):
    #print(j)
    dists = distances['pred'][j,]
    # retrieve indexes of the top (smallest distances) 'k' elements
    ind = np.argpartition(dists, k)[:k]

    # Get most freq cluster of interest
    most = [ data.obs['leiden.1.2'][x] for x  in ind ]
    mf = stats.mode(most).mode[0]
    most_freq[ prediction_100e.obs_names[j] ] = mf

    # Get the closest cluster of interest
    closest = ind[np.argmin(distances['pred'][j, ind])]        
    g = data.obs['leiden.1.2'][closest]      
    top_ones[ prediction_100e.obs_names[j] ] = g

    # increment neighbor counter
    for i in range(k):
        counts['pred'][ind[i]] += 1

data.obs['pred_counts'] = counts['pred']
prediction_100e.obs['top'] = pd.Series( top_ones )

prediction_100e.obs['freq'] = pd.Series( most_freq )
%%time
sc.write('COMBO10_SPL_lognorm_LD_prediction_hvg_LABEL_TRANSFERED', prediction_100e)
%%time
sc.write('COMBO10_NO_SPL3_Seurat3_lognorm_with_density_projections', data)<jupyter_output>CPU times: user 2min 25s, sys: 3.95 s, total: 2min 29s
Wall time: 2min 37s
{'fraction_non_alphanumeric': 0.1310534198701947, 'fraction_numerical': 0.02895656515227159, 'mean_word_length': 6.21981981981982, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '342886', 'license': 'MIT', 'n_tokens_mistral': 1542, 'n_tokens_neox': 1445, 'n_words': 292, 'path': '02_analyses/71d_PY_scGen_simulated_LD_SPL_cells_projected_on_combo10.ipynb', 'repo_name': 'elisa-laurenti/PairedOrgans', 'size': '4006'}