<jupyter_start><jupyter_text>Data import of Eils_2020/lukassen dataset<jupyter_code>import sfaira

import numpy as np
import scanpy as sc
import anndata
import pandas as pd
import matplotlib.pyplot as plt
import collections
import glob
from scipy import sparse
def subset_and_pad_adata(gene_set, adata):
    """
    This function uses a gene list provided as a Pandas dataframe with gene symbols and
    Ensembl IDs and subsets a larger Anndata object to only the genes in this list. If
    Not all genes are found in the AnnData object, then zero-padding is performed.
    """
    # Example inputs:
    # genes_filename = '/storage/groups/ml01/workspace/hlca_lisa.sikkema_malte.luecken/genes_for_mapping.csv'
    # data_filename = '/storage/groups/ml01/workspace/hlca_lisa.sikkema_malte.luecken/ready/adams.h5ad'
    # gene_set = pd.read_csv(genes_filename)
    # adata = sc.read(data_filename)

    # Prep objects
    if 'gene_symbols' in gene_set.columns:
        gene_set.index = gene_set['gene_symbols']

    else:
        raise ValueError('The input gene list was not of the expected type!\n'
                         'Gene symbols and ensembl IDs are expected in column names:\n'
                         '\t`gene_symbols` and `Unnamed: 0`')

    # Subset adata object
    common_genes = [gene for gene in gene_set['gene_symbols'].values if gene in adata.var_names]
    if len(common_genes) == 0:
        print("WARNING: YOU SHOULD PROBABLY SWITCH YOUR ADATA.VAR INDEX COLUMN TO GENE NAMES"
                  " RATHER THAN IDS! No genes were recovered.")
        return

    adata_sub = adata[:,common_genes].copy()

    # Pad object with 0 genes if needed
    if len(common_genes) < len(gene_set):
        diff = len(gene_set) - len(common_genes)
        print(f'not all genes were recovered, filling in 0 counts for {diff} missing genes...')

        # Genes to pad with
        genes_to_add = set(gene_set['gene_symbols'].values).difference(set(adata_sub.var_names))
        new_var = gene_set.loc[genes_to_add]

        if 'Unnamed: 0' in new_var.columns:
            # Assumes the unnamed column are ensembl values
            new_var['ensembl'] = new_var['Unnamed: 0']
            del new_var['Unnamed: 0']

        df_padding = pd.DataFrame(data=np.zeros((adata_sub.shape[0],len(genes_to_add))), index=adata_sub.obs_names, columns=new_var.index)
        adata_padding = sc.AnnData(df_padding, var=new_var)

        # Concatenate object
        adata_sub = anndata.concat([adata_sub, adata_padding], axis=1, join='outer', index_unique=None, merge='unique')

    # Ensure ensembl IDs are available
    adata_sub.var['ensembl'] = gene_set['Unnamed: 0']

    return adata_sub
gene_set = pd.read_csv("genes_for_mapping.csv")
gene_set<jupyter_output><empty_output><jupyter_text>Lukassen<jupyter_code>univ = sfaira.data.Universe(
    data_path="../../../../../../datasets/projects/20200101_Various_SfairaDataRepository_leander.dony/raw/",
    meta_path="../../../../../../datasets/projects/20200101_Various_SfairaDataRepository_leander.dony/meta/",
    cache_path="../../../../../../datasets/projects/20200101_Various_SfairaDataRepository_leander.dony/cache/"
)
univ.subset("doi", "10.1101/2020.03.13.991455")
univ.load()
lukassen = univ.datasets[univ.ids[0]].adata.copy()
lukassen.obs
l1dict = {"orig.ident": "sample", "nCount_RNA": "rm1", "nFeature_RNA": "rm2", "percent.mito": "rm3", "CellType": "original_celltype_ann", "smoking": "condition", "packyears": "smoking_history"}
lukassen.obs = lukassen.obs.rename(columns=l1dict)
for col in lukassen.obs.columns:
    if col.startswith("rm"):
        del lukassen.obs[col]
lukassen.obs["subject_ID"] = lukassen.obs["sample"]
lukassen.obs["study"] = "Eils2020"
lukassen.obs["dataset"] = lukassen.obs["study"]
lukassen.obs["condition"] = lukassen.obs["condition"].cat.rename_categories(['healthy', 'smoking'])
lukassen.obs["smoking_status"] = [{'healthy': "never", 'smoking': "active"}[i] for i in lukassen.obs["condition"]]
del lukassen.obsm["X_pca"], lukassen.obsm["X_umap"]
lukassen
lukassen.obs
lukassen.var
lukassen.write("/storage/groups/ml01/workspace/hlca_lisa.sikkema_malte.luecken/query_datasets/ready/full/lukassen.h5ad")
lukassen_sub = subset_and_pad_adata(gene_set, lukassen)
lukassen_sub
lukassen_sub.write("/storage/groups/ml01/workspace/hlca_lisa.sikkema_malte.luecken/query_datasets/ready/subsetted/lukassen_sub.h5ad")<jupyter_output>... storing 'name' as categorical
... storing 'gene_symbols' as categorical<jupyter_text>Szabo<jupyter_code>univ = sfaira.data.Universe(
    data_path="../../../../../../datasets/projects/20200101_Various_SfairaDataRepository_leander.dony/raw/",
    meta_path="../../../../../../datasets/projects/20200101_Various_SfairaDataRepository_leander.dony/meta/",
    cache_path="../../../../../../datasets/projects/20200101_Various_SfairaDataRepository_leander.dony/cache/"
)
univ.subset("doi", "10.1038/s41467-019-12464-3")
univ.subset("organ", "lung")
univ.load()
szabo1 = univ.datasets[univ.ids[0]].adata.copy()
szabo2 = univ.datasets[univ.ids[1]].adata.copy()
szabo3 = univ.datasets[univ.ids[2]].adata.copy()
szabo4 = univ.datasets[univ.ids[3]].adata.copy()
szabo1.obs["condition"] = univ.datasets[univ.ids[0]].state_exact
szabo2.obs["condition"] = univ.datasets[univ.ids[1]].state_exact
szabo3.obs["condition"] = univ.datasets[univ.ids[2]].state_exact
szabo4.obs["condition"] = univ.datasets[univ.ids[3]].state_exact
szabo = szabo1.concatenate(szabo2, szabo3, szabo4)
del szabo.obs["batch"]
szabo.obs["study"] = "Sims2019"
szabo.obs["dataset"] = szabo.obs["study"]
szabo.obs["sample"] = [i[:5] for i in szabo.obs.index]

szabodict = {"donor": "subject_ID", "cell_ontology_class": "original_celltype_ann"}
szabo.obs = szabo.obs.rename(columns=szabodict)
szabo = szabo[szabo.obs["original_celltype_ann"] != "unknown"].copy()
szabo.obs
szabo
szabo.var.index = szabo.var["Gene"].tolist()
szabo.var_names_make_unique()
szabo.write("../../../data/HLCA_extended/extension_datasets/ready/full/szabo.h5ad")
szabo_sub = subset_and_pad_adata(gene_set, szabo)
szabo_sub
szabo_sub.write("../../../data/HLCA_extended/extension_datasets/ready/subsetted/szabo_sub.h5ad")<jupyter_output>... storing 'Gene' as categorical
... storing 'Accession' as categorical
... storing 'gene_symbols' as categorical
{'fraction_non_alphanumeric': 0.15060432569974555, 'fraction_numerical': 0.02385496183206107, 'mean_word_length': 4.3523404255319145, 'pattern_counts': {'":': 9, '<': 12, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '76634', 'license': 'MIT', 'n_tokens_mistral': 2446, 'n_tokens_neox': 2233, 'n_words': 389, 'path': 'notebooks/3_atlas_extension/HLCA_extension_data_preprocessing/Eils_2020_lukassen.ipynb', 'repo_name': 'LungCellAtlas/HLCA_reproducibility', 'size': '6288'}