<jupyter_start><jupyter_text>A. CellphoneDB pre-processing<jupyter_code>import numpy as np
import pandas as pd
import scanpy as sc
import anndata
import os
import sys

def MovePlots(plotpattern, subplotdir):
    os.system('mkdir -p '+str(sc.settings.figdir)+'/'+subplotdir)
    os.system('mv '+str(sc.settings.figdir)+'/*'+plotpattern+'** '+str(sc.settings.figdir)+'/'+subplotdir)

sc.settings.verbosity = 1  # verbosity: errors (0), warnings (1), info (2), hints (3)
sc.settings.figdir = './cellphoneDB/'
sc.logging.print_versions()
sc.settings.set_figure_params(dpi=80)  # low dpi (dots per inch) yields small inline figures

sys.executable


def grouped_obs_percent(adata, group_key, layer=None, gene_symbols=None):
    if layer is not None:
        getX = lambda x: x.layers[layer]
    else:
        getX = lambda x: x.X
    if gene_symbols is not None:
        new_idx = adata.var[idx]
    else:
        new_idx = adata.var_names

    grouped = adata.obs.groupby(group_key)
    out = pd.DataFrame(
        np.zeros((adata.shape[1], len(grouped)), dtype=np.float64),
        columns=list(grouped.groups.keys()),
        index=adata.var_names
    )

    for group, idx in grouped.indices.items():
        X = getX(adata[idx])
        X.data = X.data > 0.01
        perc = np.asarray(np.sum(X,axis=0)/X.shape[0]).reshape(-1)
        out[group] = [round(i, 2) for i in perc ]
    return out


def grouped_obs_mean(adata, group_key, layer=None, gene_symbols=None):
    if layer is not None:
        getX = lambda x: x.layers[layer]
    else:
        getX = lambda x: x.X
    if gene_symbols is not None:
        new_idx = adata.var[idx]
    else:
        new_idx = adata.var_names

    grouped = adata.obs.groupby(group_key)
    out = pd.DataFrame(
        np.zeros((adata.shape[1], len(grouped)), dtype=np.float64),
        columns=list(grouped.groups.keys()),
        index=adata.var_names
    )

    for group, idx in grouped.indices.items():
        X = getX(adata[idx])
        out[group] = np.ravel(X.mean(axis=0, dtype=np.float64))
    return out<jupyter_output>WARNING: If you miss a compact list, please try `print_header`!<jupyter_text>Prepare INPUT Load andata<jupyter_code>adata = sc.read('/nfs/team292/lg18/with_valentina/FCA-M5-annotatedCluster4Seurat.h5ad')
adata.X.shape<jupyter_output><empty_output><jupyter_text>Load cell clusters annotation<jupyter_code># Add cluster name and doublet information
clu_annot = pd.read_csv('figures_manual_annotation/clustering_metadata.csv', header=0, index_col=0)
import collections 

if collections.Counter(adata.obs.index) == collections.Counter(clu_annot.index): 
    print ("The lists are identical") 
else : 
    print ("The lists are not identical") 
    
vars2import = ['clusters_manual','louvain', 'is_doublet', 'scrublet_cluster_score', 'scrublet_score']
for var in vars2import:
    adata.obs[var] = clu_annot[var]
meta_som = pd.read_csv('/nfs/team292/lg18/with_valentina/supporting_nocycling_annotation.csv')
meta_som = meta_som.set_index('Unnamed: 0')
meta_som.head()

meta_germ = pd.read_csv('/nfs/team292/lg18/with_valentina/germcells_annotation.csv')
meta_germ = meta_germ.set_index('Unnamed: 0')
meta_germ.head()<jupyter_output><empty_output><jupyter_text>Filter clusters of interest Use only barcodes in metadata file containing germ or somatic cells in the gonad<jupyter_code>barcodes = meta_som.index.tolist() + meta_germ.index.tolist() 
adata = adata[adata.obs.index.isin(barcodes),:]<jupyter_output><empty_output><jupyter_text>And add `cluster` label<jupyter_code>som_cell_annot = meta_som['annotated_clusters'].to_dict()
germ_cell_annot = meta_germ['annotated_clusters'].to_dict()
cell_annot = {**som_cell_annot, **germ_cell_annot}
adata.obs['clusters'] = [ cell_annot[i] for i in adata.obs_names ] 
set(adata.obs['clusters'] )<jupyter_output><empty_output><jupyter_text>Add sex info in the bipotent clusters - to match microenviroments file<jupyter_code>s = adata.obs['clusters'].isin(['sLGR5'])
idx = s[s].index.values
adata.obs['clusters_sex'] =  adata.obs['clusters']
for i in idx:
    adata.obs.at[i,'clusters_sex'] = adata.obs['clusters'][i]+'_'+adata.obs['sex'][i]
set(adata.obs['clusters_sex'])<jupyter_output><empty_output><jupyter_text>Filter cells in the microenviroments of interest<jupyter_code>micro = pd.read_csv('cellphoneDB/input/microenviroments/microenviroments.csv')
# micro = micro.set_index('Unnamed: 0')
micro.head()
cells_common = list(set(adata.obs['clusters_sex'] )  & set(micro.celltype)) 
adata = adata[[ i in cells_common for i in adata.obs.clusters_sex ]]
set(adata.obs['clusters_sex'] )
# pd.DataFrame(adata.obs['clusters_sex']).to_csv(str(sc.settings.figdir)+'/input/metadata.csv')<jupyter_output><empty_output><jupyter_text>Compute average and percentatge expression per cluster<jupyter_code># Normalized average
adata_norm = adata.copy()
sc.pp.normalize_per_cell(adata_norm, counts_per_cell_after=1e4)
sc.pp.log1p(adata_norm)
means = grouped_obs_mean(adata_norm, 'clusters_sex')
# percentatge expression
percent = grouped_obs_percent(adata, 'clusters_sex')
# # un-byte
# means.index = [ i.decode("utf-8") for i in means.index]
# percent.index = [ i.decode("utf-8") for i in percent.index]
means.head()
percent.head()
pd.DataFrame(means).to_csv(str(sc.settings.figdir)+'/input/average_log.csv')
pd.DataFrame(percent).to_csv(str(sc.settings.figdir)+'/input/percent.csv')<jupyter_output><empty_output><jupyter_text>Save gene expression Raw counts<jupyter_code>adata.write(str(sc.settings.figdir)+'/input/counts.h5ad')<jupyter_output>/opt/conda/lib/python3.8/site-packages/anndata/_core/anndata.py:1207: ImplicitModificationWarning: Initializing view as actual.
  warnings.warn(
Trying to set attribute `.obs` of view, copying.
... storing 'clusters' as categorical
Trying to set attribute `.obs` of view, copying.
... storing 'clusters_manual' as categorical
Trying to set attribute `.obs` of view, copying.
... storing 'clusters_sex' as categorical<jupyter_text>Save meta<jupyter_code>df_meta = pd.DataFrame(data={'Cell':list(adata.obs.index),
                             'cell_type':[ 'celltype_'+str(i) for i in adata.obs['clusters_sex']] })
df_meta.set_index('Cell', inplace=True)
df_meta.to_csv('cellphoneDB/input/meta.tsv', sep = '\t')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13989802421924794, 'fraction_numerical': 0.009719566602931804, 'mean_word_length': 5.088263821532493, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '339586', 'license': 'MIT', 'n_tokens_mistral': 2348, 'n_tokens_neox': 2262, 'n_words': 412, 'path': 'Cph1a_bothSex_preprocesing.ipynb', 'repo_name': 'ventolab/HGDA', 'size': '6276'}