<jupyter_start><jupyter_text>Imports<jupyter_code>import scanpy as sc
import pandas as pd
import torch
import numpy

import fiberedae.models.fae as vmod
import fiberedae.utils.basic_trainer as vtrain
import fiberedae.utils.persistence as vpers
import fiberedae.utils.single_cell as vsc
import fiberedae.utils.nn as vnnutils
import fiberedae.utils.plots as vplots
import fiberedae.utils.fae as vfae
import fiberedae.utils.datasets as vdatasets

from anndata import AnnData

import fiberedae.utils.useful as us<jupyter_output>/home/tariq/miniconda3/envs/fae/lib/python3.7/site-packages/anndata-0.7.1-py3.7.egg/anndata/_core/anndata.py:21: FutureWarning: pandas.core.index is deprecated and will be removed in a future version.  The public classes are available in the top-level namespace.
  from pandas.core.index import RangeIndex<jupyter_text>Kang INF Beta stimulated cellsThis dataset contains batches of stimulated and stimualated PBMCs.  Loading dataset and configuration<jupyter_code>train_hv = sc.read("./datasets/kang_normalized_2khv.h5ad", backup_url="https://www.dropbox.com/s/8jq33luekpdnytt/kang_normalized_2khv.h5ad?dl=1")<jupyter_output><empty_output><jupyter_text>Original uncorrected dataset  Training and saving modelIn this section we train the model on the full gene expression matrix  Loading model and configuration<jupyter_code>train_hv.X = train_hv.X - numpy.min(train_hv.X)
train_hv.X = train_hv.X / numpy.max(train_hv.X)
sc.pp.pca(train_hv)
sc.pp.neighbors(train_hv)
sc.tl.umap(train_hv)
sc.pl.umap(train_hv, color=["condition", "cell_type"], wspace=.5, frameon=False)
config = us.load_configuration("configurations/kang_pbmc.json")
dataset = vdatasets.make_single_cell_dataset(
    batch_size=config["hps"]["minibatch_size"],
    condition_field=config["dataset"]["condition_field"],
    adata=train_hv,
    dataset_name=config["dataset"]["name"],
    oversample=True
)
config
model_args = dict(config["model"])
model_args.update(
    dict(
        x_dim=dataset["shapes"]["input_size"],
        nb_class=dataset["shapes"]["nb_class"],
        output_transform=vnnutils.ScaleNonLinearity(-1., 1., dataset["sample_scale"][0], dataset["sample_scale"][1]),
    )
)
#make model
model = vmod.FiberedAE(**model_args)
model.to("cuda")

#train model
trainer, history = us.train(model, dataset, config, nb_epochs=config["hps"]["nb_epochs"])
vpers.save_model(model, {}, history, {}, dataset["label_encoding"], model_args,
                 "./trained_models/%s_%se.pytorch.mdl" % (
                     config["dataset"]["dataset_name"].replace(" ", "_"),
                     config["hps"]["nb_epochs"]
                 )
                )
#model = vpers.load_model("./trained_models/kang_pbmc_16_hv_2000e.pytorch.mdl", vmod.FiberedAE, "cpu")[0]<jupyter_output><empty_output><jupyter_text>Learning curves<jupyter_code>vplots.interactive_line_plots(
    dct_values = {
        "G": history["train"]["gan_discriminator"],
        "GD": history["train"]["gan_generator"]
    },
    x_label = "epochs",
    y_label = "loss",
)<jupyter_output><empty_output><jupyter_text>Latent space plots<jupyter_code>latent_space = vfae.get_latent_space(
    model=model,
    data_loader=dataset["loaders"]["train"],
    batch_formater=dataset["batch_formater"],
    label_encoding=dataset["label_encoding"]
)
vplots.plot_latent(latent_space["base_space"], latent_space["str_labels"], point_opacity=1 )
vplots.plot_latent(latent_space["fiber_space"], latent_space["str_labels"])<jupyter_output><empty_output><jupyter_text>Batch correctionHere we correct the batch effect by using the model to translate all gene expressions into the domain of a single reference batch. In all subesquent figures we see that, post-translation cells from different batches are well mixed and that cell now cluster by cell types instead of batch. Here it is important to note that the model had no explicit knowledge of cell types during training.  Taking control as reference<jupyter_code>adjusted_adata = vsc.translate(model, train_hv, "condition", 'control', dataset["label_encoding"], batch_size=1024)
adjusted_adata.write("./fae_naive_16_ctrl.h5ad")
sc.pp.pca(adjusted_adata)
sc.pp.neighbors(adjusted_adata)
sc.tl.umap(adjusted_adata)
sc.pl.umap(adjusted_adata, color=["condition", "cell_type"], wspace=.5, frameon=False)<jupyter_output><empty_output><jupyter_text>Taking stimulated as reference<jupyter_code>adjusted_adata = vsc.translate(model, train_hv, "condition", 'stimulated', dataset["label_encoding"], batch_size=1024)

sc.pp.pca(adjusted_adata)
sc.pp.neighbors(adjusted_adata)
sc.tl.umap(adjusted_adata)
adjusted_adata.write("./fae_naive_16_stim.h5ad")
sc.pl.umap(adjusted_adata, color=["condition", "cell_type"], wspace=.5, frameon=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12355536877495273, 'fraction_numerical': 0.00987602437486867, 'mean_word_length': 6.020648967551622, 'pattern_counts': {'":': 2, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '48634', 'license': 'Apache-2.0', 'n_tokens_mistral': 1682, 'n_tokens_neox': 1637, 'n_words': 301, 'path': 'demos/kang_pbmc_16_hv.ipynb', 'repo_name': 'tariqdaouda/FiberedAE', 'size': '4759'}