<jupyter_start><jupyter_text>Summary of reconstruction metric for different technologies<jupyter_code>import pickle
import seaborn as sb
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import json<jupyter_output><empty_output><jupyter_text>Table for Average KS statistics and average deviance<jupyter_code>import pickle
import seaborn as sb
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import json
truncated = True
experimental_name = {
                     0:'jurkat_293t_50_50',
                     8:'jurkat_293t_99_1',
                     9: 'Zheng_jurkat',
                     10:'Zheng_293t',
                     1:'pancreas_inDrop', 2:'pancreas_multi_celseq2_expression_matrix',
                     3:'pancreas_multi_fluidigmc1_expression_matrix', 4:'pancreas_multi_smartseq2_expression_matrix',
                     5:'pancreas_multi_celseq_expression_matrix',
                     }

gene_index = pd.Index(['25000'])
Average_deviance = dict()
for experimental_no in range(0,6):
    Frob = []
    KL = [] 
    KS_cell = []
    KS_gene = []
    for i in gene_index:
        infile = '../Result/benchmark/Dataset_{}_top{}_variance.json'.format(experimental_name[experimental_no],i)
        with open(infile) as f: 
            temp = json.load(f)
            Frob.append(temp['Frob'])
            KL.append(temp['KL'])
            KS_cell.append(temp['KS_cell'])
            KS_gene.append(temp['KS_gene'])
    Frob = pd.DataFrame(Frob,index=gene_index)
    KL = pd.DataFrame(KL,index=gene_index)
    KS_cell = pd.DataFrame(KS_cell,index=gene_index)
    KS_gene = pd.DataFrame(KS_gene,index=gene_index)
    if truncated:
        # Frobenius norm 
        # reshape dataframe into 
        all_df = (Frob.loc[:,['PCA_10','NMF_10','NMFP_10','HPF_10']] + KL.loc[:,['PCA_10','NMF_10','NMFP_10','HPF_10']])/2
    else:
        all_df = (Frob.iloc[:,4:8] + KL.iloc[:,4:8])/2
    Average_deviance[experimental_name[experimental_no]] = all_df.iloc[-1,:]
    
np.round(pd.DataFrame(Average_deviance),3)
np.round(np.mean(pd.DataFrame(Average_deviance),axis=1),3)
Average_KS_Statistics = dict()
for experimental_no in range(0,6):
    Frob = []
    KL = [] 
    KS_cell = []
    KS_gene = []
    for i in gene_index:
        infile = '../Result/benchmark/Dataset_{}_top{}_variance.json'.format(experimental_name[experimental_no],i)
        with open(infile) as f: 
            temp = json.load(f)
            Frob.append(temp['Frob'])
            KL.append(temp['KL'])
            KS_cell.append(temp['KS_cell'])
            KS_gene.append(temp['KS_gene'])
    Frob = pd.DataFrame(Frob,index=gene_index)
    KL = pd.DataFrame(KL,index=gene_index)
    KS_cell = pd.DataFrame(KS_cell,index=gene_index)
    KS_gene = pd.DataFrame(KS_gene,index=gene_index)
    if truncated:
        # Frobenius norm 
        # reshape dataframe into 
        all_df = (KS_cell.loc[:,['PCA_10','NMF_10','NMFP_10','HPF_10']] + KS_gene.loc[:,['PCA_10','NMF_10','NMFP_10','HPF_10']])/2
    else:
        all_df = (KS_cell.iloc[:,4:8] + KS_gene.iloc[:,4:8])/2
    Average_KS_Statistics[experimental_name[experimental_no]] = all_df.iloc[-1,:]
    
np.round(pd.DataFrame(Average_KS_Statistics),3)
np.round(np.mean(pd.DataFrame(Average_KS_Statistics),axis=1),3)<jupyter_output><empty_output><jupyter_text>Hyper-parameters<jupyter_code># Hyper-parameters 
import pickle
import seaborn as sb
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import json
# sb.set(style="white", palette="muted", color_codes=True)

a = np.linspace(0.1,2.1,num=11)
c = np.linspace(0.1,2.1,num=11)
gene_index = pd.MultiIndex.from_product([a, c],names=['a','c'])
df = pd.DataFrame(index=gene_index)
experimental_no = 5
experimental_name = {
                     0:'pancreas_inDrop', 1:'pancreas_multi_celseq2_expression_matrix',
                     2:'pancreas_multi_fluidigmc1_expression_matrix', 3:'pancreas_multi_smartseq2_expression_matrix',
                     4:'pancreas_multi_celseq_expression_matrix', 5:'jurkat_293t_50_50',
                     }
DatasetID = {
            0:6, 1:4, 2:7, 3:5, 4:8, 5:3
}



for experimental_no in range(0,6):
    Frob = []
    KL = [] 
    KS_cell = []
    KS_gene = []
    for i in gene_index:
        infile = '../Result/{}_hyper/Dataset_{}_hyper_{}_{}.json'.format(
            experimental_name[experimental_no],experimental_name[experimental_no],i[0],i[1])
        with open(infile) as f: 
            temp = json.load(f)
            Frob.append(temp['Frob'])
            KL.append(temp['KL'])
            KS_cell.append(temp['KS_cell'])
            KS_gene.append(temp['KS_gene'])
    Frob = pd.DataFrame(Frob,index=gene_index)
    KL = pd.DataFrame(KL,index=gene_index)
    KS_cell = pd.DataFrame(KS_cell,index=gene_index)
    KS_gene = pd.DataFrame(KS_gene,index=gene_index)
    
    fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2)
    fig.set_size_inches(18.5, 10.5, forward=True)
    fig.suptitle('Kolmogorov-Smirnov statistics for Dataset {}'.format(DatasetID[experimental_no]))
    KS_cell.groupby('a').mean().plot(ax=ax1)
    ax1.set(xlabel='cell scale hyper-parameter a', ylabel='cell KS statistic')
    ax1.set_title("(a)", fontsize=14, y=-0.2, ha='center')
    KS_cell.groupby('c').mean().plot(ax=ax2)
    ax2.set(xlabel='gene scale hyper-parameter c', ylabel='cell KS statistic')
    ax2.set_title("(b)", fontsize=14, y=-0.2, ha='center')
    KS_gene.groupby('a').mean().plot(ax=ax3,ylim=[np.min(KS_gene.values),np.max(KS_gene.values)+0.001])
    ax3.set(xlabel='cell scale hyper-parameter a', ylabel='gene KS statistic')
    ax3.set_title("(c)", fontsize=14, y=-0.2, ha='center')
    KS_gene.groupby('c').mean().plot(ax=ax4,ylim=[np.min(KS_gene.values),np.max(KS_gene.values)+0.001])
    ax4.set(xlabel='gene scale hyper-parameter c', ylabel='gene KS statistic')
    ax4.set_title("(d)", fontsize=14, y=-0.2, ha='center')
    plt.savefig('../Result/Dataset_{}_hyper.pdf'.format(DatasetID[experimental_no]),format='pdf')<jupyter_output><empty_output><jupyter_text>Summary of reconstruction metrics for a specific dataset (10 latent factors)<jupyter_code>import matplotlib.pyplot as plt
experimental_no = 5
truncated = True
experimental_name = {
                     0:'pancreas_inDrop', 1:'pancreas_multi_celseq2_expression_matrix',
                     2:'pancreas_multi_fluidigmc1_expression_matrix', 3:'pancreas_multi_smartseq2_expression_matrix',
                     4:'pancreas_multi_celseq_expression_matrix', 5:'jurkat_293t_50_50',
                     }
DatasetID = {
            0:6, 1:4, 2:7, 3:5, 4:8, 5:3
}
gene_index = pd.Index([2500,5000,10000,15000,17500,25000])
for experimental_no in range(0,6):

    fig, ((ax1, ax2)) = plt.subplots(1, 2)
    fig.set_size_inches(12.5, 5.5, forward=True)
    fig.suptitle('Effect of pre-processing on dataset {}'.format(DatasetID[experimental_no]))

    Frob = []
    KL = [] 
    KS_cell = []
    KS_gene = []
    for i in gene_index:
        infile = '../Result/benchmark/Dataset_{}_top{}_variance.json'.format(experimental_name[experimental_no],i)
        with open(infile) as f: 
            temp = json.load(f)
            Frob.append(temp['Frob'])
            KL.append(temp['KL'])
            KS_cell.append(temp['KS_cell'])
            KS_gene.append(temp['KS_gene'])
    Frob = pd.DataFrame(Frob,index=gene_index)
    KL = pd.DataFrame(KL,index=gene_index)
    KS_cell = pd.DataFrame(KS_cell,index=gene_index)
    KS_gene = pd.DataFrame(KS_gene,index=gene_index)
    if truncated:
        # Frobenius norm 
        # reshape dataframe into 
        frob_df = Frob.loc[:,['PCA_10','NMF_10','NMFP_10','HPF_10']]
        kl_df = KL.loc[:,['PCA_10','NMF_10','NMFP_10','HPF_10']]
        cell_df = (Frob.loc[:,['PCA_10','NMF_10','NMFP_10','HPF_10']] + KL.loc[:,['PCA_10','NMF_10','NMFP_10','HPF_10']])/2
        gene_df = (KS_cell.loc[:,['PCA_10','NMF_10','NMFP_10','HPF_10']] + KS_gene.loc[:,['PCA_10','NMF_10','NMFP_10','HPF_10']])/2
    else:
        gene_df = KS_gene.loc[:,['PCA_10','NMF_10','NMFP_10','HPF_10']]

    cell_df.plot(ax=ax1)
    ax1.set(xlabel='Number of genes M', ylabel='Average explained deviance',title='Explained Deviance')
    ax1.legend(['PCA','NMF','NMF Poisson','HPF'])
    gene_df.plot(ax=ax2)
    ax2.set(xlabel='Number of genes M', ylabel='Average KS statistics',title='Average KS statistics' )
    ax2.legend(['PCA','NMF','NMF Poisson','HPF'])

    plt.savefig('../Result/Dataset_{}_preprocessing_metrics.pdf'.format(DatasetID[experimental_no]),format='pdf')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15574249328192546, 'fraction_numerical': 0.037037037037037035, 'mean_word_length': 3.6071044133476855, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '541484', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 3451, 'n_tokens_neox': 3103, 'n_words': 440, 'path': 'IHPF/HPF_robustness.ipynb', 'repo_name': 'barahona-research-group/scIHPF', 'size': '8559'}