<jupyter_start><jupyter_text>TCGA-BRCA (The Cancer Genome Atlas - Breast Cancer)<jupyter_code>tcga = pd.read_csv(bkdata_path, sep='\t')
tcga = tcga.sort_index()
tcga.to_csv('../DATA/TCGA/TCGA_GDC_HTSeq_TPM.csv')
abis = pd.read_csv('../DATA/GSE107011/GSE107011_Processed_data_TPM.txt')
epic = pd.read_csv('../DATA/EPIC/melanoma_counts.csv')<jupyter_output><empty_output><jupyter_text>METABRIC (Molecular<jupyter_code>metabric = pd.read_csv('../DATA/METABRIC/data_expression_median.txt', sep='\t', index_col=0)
metabric.index.name = None
metabric = metabric.iloc[:,1:]
metabric = metabric.sort_index()
metabric.to_csv('../DATA/METABRIC/METABRIC.csv')
sdy67_1 = pd.read_csv('../DATA/SDY67/SDY67_EXP13377_RNA_seq.703318.tsv', sep='\t', index_col=0)
sdy67_2 = pd.read_csv('../DATA/SDY67/SDY67_EXP14625_RNA_seq.703317.tsv', sep='\t', index_col=0)
sdy67_1.index.name = None
sdy67_2.index.name = None
sdy67 = sdy67_1.join(sdy67_2)
sdy67 = sdy67.sort_index()
sdy67_meta = pd.read_csv('../DATA/SDY67/SDY67-DR34_Subject_2_RNA_sequencing_result.txt', sep='\t')
sdy67_meta['SubjectID'] = [sub+'_'+str(time) for sub, time in zip(list(sdy67_meta['Subject Accession'].values),list(sdy67_meta['Study Time Collected'].values))]
sdy67_id = sdy67_meta.reset_index().set_index('Expsample Accession').loc[sdy67.columns.values.tolist(),'SubjectID'].values.tolist()
sdy67.columns = sdy67_id
sdy67
# sdy67_label = pd.read_csv('../DATA/SDY67/SDY67_250_label.csv', index_col=0) #'../DATA/SDY67/SDY67_extracted_from_mmc7.csv', index_col=0)
sdy67_label = pd.read_csv('../DATA/SDY67/SDY67_raw_labels_from_mmc7.csv', index_col=0) #'../DATA/SDY67/SDY67_extracted_from_mmc7.csv', index_col=0)
sdy67_label.index.name = None
sdy67_label['Other'] = 0
# sdy67_label = sdy67_label.fillna(0)
sdy67_label = sdy67_label.dropna()
sdy67_label = sdy67_label.reindex(sdy67.columns)
for i in range(len(sdy67_label)):
    sumval = np.sum(sdy67_label.iloc[i,:])
    if sumval >= 100.:
        sdy67_label.iloc[i,:] = sdy67_label.iloc[i,:]/sumval
#     elif sumval == 0.:
#         sdy67_label.iloc[i,:] = None
    else:
        sdy67_label.iloc[i,-1] = 100.-sumval
        sdy67_label.iloc[i,:] = sdy67_label.iloc[i,:]/100.

sdy67_label = sdy67_label.dropna()
sdy67_label
sdy67 = sdy67.loc[:,sdy67_label.index]
sdy67
sdy67.to_csv('../DATA/SDY67/SDY67_468_pp.csv')
sdy67_label.to_csv('../DATA/SDY67/SDY67_468_label_allct_pp.csv')
sdy67_label2 =  pd.DataFrame()
sdy67_label2['B cell'] = sdy67_label['B Naive'] + sdy67_label['B Ex'] + sdy67_label['B NSM'] + sdy67_label['B SM'] + sdy67_label['Plasmablasts']
sdy67_label2['T CD4'] = sdy67_label['T CD4']
sdy67_label2['T CD8'] = sdy67_label['T CD8']
sdy67_label2['NK'] = sdy67_label['NK']
sdy67_label2['Myeloid'] = sdy67_label['Basophils LD'] + sdy67_label['mDCs'] + sdy67_label['pDCs'] + sdy67_label['Monocytes C'] + sdy67_label['Monocytes I'] + sdy67_label['Monocytes NC']
sdy67_label2['Other'] = sdy67_label['Other']

sdy67_label2
# sdy67.to_csv('../DATA/SDY67/SDY67_468_pp.csv')
sdy67_label2.to_csv('../DATA/SDY67/SDY67_250_label_6ct_pp.csv')<jupyter_output><empty_output><jupyter_text>-----------------------------------------------------------------------------<jupyter_code>y = ad.read_h5ad('C:/Users/yw_ji/Documents/MSc Thesis/DATA/scaden_input/simulated_data/data_10000.h5ad')
y_data = y_data.reindex(y.var.index)
sdy67_ad = ad.AnnData(sdy67.T, obs=sdy67_label)
sdy67_ad.uns['cell_types'] = sdy67_label.columns.tolist()
sdy67_ad = sdy67_ad[:,y.var.index]
sdy67_ad.write_h5ad('C:/Users/yw_ji/Documents/MSc Thesis/DATA/scaden_input/SDY67_250.h5ad')
sdy67_ad.var_names
CELLTYPES = ['CAF','B_cell','Cancer','Endothelial','Myeloid','Other_T','CD8+_T_cell',
             'NK_cell','CD4+_T_cell','Treg','PVL','Epithelial']
sdy67_label.dropna().index
# ABIS dataset need to load genes.tsv then match Ensembl id to gene id 
# EPIC dataset needs fixing
# abis = pd.read_csv('../DATA/GSE107011/GSE107011_Processed_data_TPM.txt', index_col=0) # EnsDb.Hsapiens.v79, aggregated signals from duplicate probes by max value
abis = pd.read_csv('../DATA/GSE107011/GSE107011_Processed_data_TPM_HGNC_Gene_ID.csv', index_col=0)
abis_label = pd.read_csv('../DATA/GSE107011/abis_raw_labels_from_mmc7.csv', index_col=0)
# abis_label = pd.read_csv('../DATA/GSE107011/GSE107011_label_7ct.csv', index_col=0)
for i in range(len(abis_label)):
    sumval = np.sum(abis_label.iloc[i,:])
    if sumval >= 100.:
        abis_label.iloc[i,:] = abis_label.iloc[i,:]/sumval
    else:
        abis_label.iloc[i,-1] += 100.-sumval
        abis_label.iloc[i,:] = abis_label.iloc[i,:]/100.
abis = abis.drop(columns='DZQV_PBMC')
abis_pbmc = abis.filter(regex='_PBMC$', axis=1)
abis_pbmc_label = abis_label
### June 13, 2021, ABIS 6 celltypes
abis_pbmc_label2 =  pd.DataFrame()
abis_pbmc_label2['B cell'] = abis_pbmc_label['B Naive'] + abis_pbmc_label['B Ex'] + abis_pbmc_label['B NSM'] + abis_pbmc_label['B SM'] + abis_pbmc_label['Plasmablasts']
abis_pbmc_label2['T CD4'] = abis_pbmc_label['T CD4']
abis_pbmc_label2['T CD8'] = abis_pbmc_label['T CD8']
abis_pbmc_label2['NK'] = abis_pbmc_label['NK']
abis_pbmc_label2['Myeloid'] = abis_pbmc_label['Basophils LD'] + abis_pbmc_label['mDCs'] + abis_pbmc_label['pDCs'] + abis_pbmc_label['Monocytes C'] + abis_pbmc_label['Monocytes I'] + abis_pbmc_label['Monocytes NC']
abis_pbmc_label2['Other'] = abis_pbmc_label['Other']
abis_pbmc_label2.to_csv('../DATA/GSE107011/abis_pbmc_label_6ct.csv')
sdy67_label.columns
abis_label.columns
pd.Series([x[5:] for x in abis.columns.tolist()]).unique()
abis_label
row_names = []
values = []

for i, j in enumerate(abis.columns):
    if j[5:] == 'B_naive':
        celltype = 'B Naive'
        idx = 0
    elif j[5:] == 'B_Ex':
        celltype = 'B Ex'
        idx = 1   
    elif j[5:] == 'B_NSM':
        celltype = 'B NSM'
        idx = 2
    elif j[5:] == 'B_SM':
        celltype = 'B SM'
        idx = 3
    elif j[5:] == 'Plasmablasts':
        celltype = 'Plasmablasts'
        idx = 4
    elif j[5:] in ['TFH','Treg','Th1','Th1/Th17','Th17','Th2','CD4_naive', 'CD4_TE']:
        celltype = 'T CD4'
        idx = 5
    elif j[5:] in ['CD8_naive','CD8_CM','CD8_EM','CD8_TE']:
        celltype = 'T CD8'
        idx = 6
    elif j[5:] == 'Basophils':
        celltype = 'Basophils LD'
        idx = 7
    elif j[5:] == 'mDC':
        celltype = 'mDCs'
        idx = 8
    elif j[5:] == 'pDC':
        celltype = 'pDCs'
        idx = 9
    elif j[5:] == 'C_mono':
        celltype = 'Monocytes C'
        idx = 10
    elif j[5:] == 'I_mono':
        celltype = 'Monocytes I'
        idx = 11
    elif j[5:] == 'NC_mono':
        celltype = 'Monocytes NC'
        idx = 12
    elif j[5:] == 'NK':
        celltype = 'NK'
        idx = 13
    elif j[5:] in ['MAIT','VD2+','VD2-','Progenitor','Neutrophils']:
        celltype = 'Other'
        idx = 14
    else:
        celltype = 'PBMC'
        value = [abis_label.loc[[j[:4]],['B Naive']].values.sum(),
                 abis_label.loc[[j[:4]],['B Ex']].values.sum(),
                 abis_label.loc[[j[:4]],['B NSM']].values.sum(),
                 abis_label.loc[[j[:4]],['B SM']].values.sum(),
                 abis_label.loc[[j[:4]],['Plasmablasts']].values.sum(),
                 abis_label.loc[[j[:4]],['Tfh','Tregs','Th1','Th1/Th17','Th17','Th2','T CD4 Naive', 'T CD4 TE']].values.sum(),
                 abis_label.loc[[j[:4]],['T CD8 Naive','T CD8 CM','T CD8 EM','T CD8 TE']].values.sum(),
                 abis_label.loc[[j[:4]],['Basophils LD']].values.sum(),
                 abis_label.loc[[j[:4]],['mDCs']].values.sum(),
                 abis_label.loc[[j[:4]],['pDCs']].values.sum(),
                 abis_label.loc[[j[:4]],['Monocytes C']].values.sum(),
                 abis_label.loc[[j[:4]],['Monocytes I']].values.sum(),
                 abis_label.loc[[j[:4]],['Monocytes NC']].values.sum(),
                 abis_label.loc[[j[:4]],['NK']].values.sum(),
                 abis_label.loc[[j[:4]],['MAIT','T gd Vd2','T gd non-Vd2','Progenitors','Neutrophils LD']].values.sum(),
                ]

    if j == 'DZQV_PBMC':
        pass
    else:
        sample_name = j
        row_names.append(sample_name)
        tmp = [0]*15
        if celltype == 'PBMC':
            values.append(value)
        else:
            tmp[idx] = 1
            values.append(tmp)

abis_label2 = pd.DataFrame(data=values, index=row_names, columns=sdy67_label.columns)
row_names = []
values = []

for i, j in enumerate(abis.columns):
    if j[5:] in ['B_naive','B_NSM','B_Ex','B_SM','Plasmablasts']:
        celltype = 'B_cell'
        idx = 0
    elif j[5:] in ['TFH','Treg','Th1','Th1/Th17','Th17','Th2','CD4_naive', 'CD4_TE']:
        celltype = 'T_CD4'
        idx = 1
    elif j[5:] in ['CD8_naive','CD8_CM','CD8_EM','CD8_TE']:
        celltype = 'T_CD8'
        idx = 2
    elif j[5:] in ['pDC', 'mDC']:
        celltype = 'DCs'
        idx = 3
    elif j[5:] in ['C_mono','I_mono','NC_mono']:
        celltype = 'Monocytes'
        idx = 4
    elif j[5:] in ['NK']:
        celltype = 'NK'
        idx = 5
    elif j[5:] in ['MAIT','VD2+','VD2-','Progenitor','Neutrophils','Basophils']:
        celltype = 'Other'
        idx = 6
    else:
        celltype = 'PBMC'

    if j == 'DZQV_PBMC':
        pass
    else:
        sample_name = j
        row_names.append(sample_name)
        tmp = [0]*7
        if celltype == 'PBMC':
            values.append(abis_label.loc[[sample_name]].values.squeeze().tolist())
        else:
            tmp[idx] = 1
            values.append(tmp)

abis_label = pd.DataFrame(data=values, index=row_names, columns=abis_label.columns)
abis_label
# abis.to_csv('../DATA/GSE107011/abis_data.csv')
abis_label.to_csv('../DATA/GSE107011/abis_label_15ct.csv')
abis_pbmc_label['T CD4'] = abis_pbmc_label.loc[:,['Tfh','Tregs','Th1','Th1/Th17','Th17','Th2','T CD4 Naive', 'T CD4 TE']].sum(axis=1)
abis_pbmc_label['T CD8'] = abis_pbmc_label.loc[:,['T CD8 Naive','T CD8 CM','T CD8 EM','T CD8 TE']].sum(axis=1)
abis_pbmc_label['Other'] = abis_pbmc_label.loc[:,['MAIT','T gd Vd2','T gd non-Vd2','Progenitors','Neutrophils LD']].sum(axis=1)
abis_pbmc_label = abis_pbmc_label.loc[:,['B Naive','B Ex','B NSM','B SM','Plasmablasts','T CD4','T CD8','Basophils LD','mDCs','pDCs','Monocytes C','Monocytes I','Monocytes NC','NK','Other']]
# abis_pbmc.to_csv('../DATA/GSE107011/abis_pbmc_data.csv')
abis_pbmc_label.to_csv('../DATA/GSE107011/abis_pbmc_label_15ct.csv')
abis_pbmc_label
abis_label2
np.sum(abis_label2, axis=1)
abis
abis[abis.columns[abis.columns.to_series().str.contains('_PBMC')]]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1931106471816284, 'fraction_numerical': 0.0556082748149554, 'mean_word_length': 3.655035335689046, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '173675', 'license': 'MIT', 'n_tokens_mistral': 4829, 'n_tokens_neox': 4386, 'n_words': 612, 'path': 'notebooks/data_bulk.ipynb', 'repo_name': 'ywjin0707/NNICE', 'size': '10538'}