<jupyter_start><jupyter_text>import scVI models<jupyter_code>from scvi.inference import UnsupervisedTrainer, SemiSupervisedTrainer
from scvi.models.scanvi import SCANVI
from scvi.models.vae import VAE
import numpy as np
gene_dataset = ZeiselMoleArchData()
gene_dataset_fine = ZeiselMoleArchData(coarse=False)
print("The number of classes in the coarse data is %i\n" % len(np.unique(gene_dataset.labels)))
print("The number of classes in the fine data is %i\n" % len(np.unique(gene_dataset_fine.labels)))
if np.sum([gene_dataset_fine.labels_groups[x] for x in gene_dataset_fine.labels.ravel()] == gene_dataset.labels.ravel()) == len(gene_dataset_fine):
    print("the mapping of fine to coarse labels is Correct\n") 
else: print("check that labels_groups is right")
     

import pandas as pd
genes1 = pd.read_table('../Zeisel/Zeisel.1.hvg_info.csv', delimiter=',')
geneid1 = np.asarray([x.replace('gene_', '') for x in genes1[genes1.keys()[0]]]).astype('int')
genenames1 = genes1['genename']
assert np.sum(np.asarray(genenames1) == gene_dataset.gene_names) == len(gene_dataset.gene_names)
genes = gene_dataset.gene_names[geneid1[:1000]]
from scvi.harmonization.utils_chenling import subsetByGenenames
gene_dataset = subsetByGenenames(gene_dataset,genes)
gene_dataset_fine = subsetByGenenames(gene_dataset_fine,genes)
vae = VAE(gene_dataset.nb_genes, n_batch=gene_dataset.n_batches, n_labels=gene_dataset.n_labels,
          n_hidden=128, n_latent=10, n_layers=2, dispersion='gene')
trainer = UnsupervisedTrainer(vae, gene_dataset, train_size=1.0)
trainer.model.load_state_dict(torch.load('../Zeisel/vae.pkl'))
trainer.model.eval()
# trainer.train(n_epochs=250)
# torch.save(trainer.model.state_dict(),'../Zeisel/vae.pkl')
full = trainer.create_posterior(trainer.model, gene_dataset, indices=np.arange(len(gene_dataset)))<jupyter_output><empty_output><jupyter_text>SCANVI Fine<jupyter_code>scanvi = SCANVI(gene_dataset_fine.nb_genes, gene_dataset_fine.n_batches, gene_dataset_fine.n_labels, n_layers=2)
scanvi.load_state_dict(full.model.state_dict(), strict=False)
trainer_scanvi = SemiSupervisedTrainer(scanvi, gene_dataset_fine, classification_ratio=50,
                                       n_epochs_classifier=1, lr_classification=5 * 1e-3)
trainer_scanvi.train(n_epochs=5)
torch.save(trainer_scanvi.model.state_dict(),'../Zeisel/scanvi.pkl')
full = trainer_scanvi.create_posterior(trainer_scanvi.model, gene_dataset_fine, indices=np.arange(len(gene_dataset_fine)))
y,pred = full.sequential().compute_predictions()
np.mean(y==pred)<jupyter_output><empty_output><jupyter_text>SCANVI Hierarchical<jupyter_code>scanvi = SCANVI(gene_dataset_fine.nb_genes, gene_dataset_fine.n_batches, gene_dataset_fine.n_labels, n_layers=2,
               labels_groups=gene_dataset_fine.labels_groups, use_labels_groups=True)

scanvi.load_state_dict(full.model.state_dict(), strict=False)


trainer_scanvi = SemiSupervisedTrainer(scanvi, gene_dataset_fine, classification_ratio=50,
                                       n_epochs_classifier=1, lr_classification=5 * 1e-3)

trainer_scanvi.train(n_epochs=5)

full = trainer_scanvi.create_posterior(trainer_scanvi.model, gene_dataset_fine, indices=np.arange(len(gene_dataset_fine)))
y,pred = full.sequential().compute_predictions()
np.mean(y==pred)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12054380664652568, 'fraction_numerical': 0.012386706948640483, 'mean_word_length': 6.997584541062802, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '481826', 'license': 'MIT', 'n_tokens_mistral': 1271, 'n_tokens_neox': 1173, 'n_words': 164, 'path': 'notebooks/.ipynb_checkpoints/Hierarchical-checkpoint.ipynb', 'repo_name': 'chenlingantelope/HarmonizationSCANVI', 'size': '3310'}