<jupyter_start><jupyter_text>Full xena file<jupyter_code>import pandas as pd
import numpy as np
from sklearn.model_selection import StratifiedShuffleSplit
import os
ge = np.load("/home/bram/jointomicscomp/data/GE.npy")
me = np.load("/home/bram/jointomicscomp/data/ME.npy")
gcn = np.load("/home/bram/jointomicscomp/data/GCN.npy")
samples = np.load("/home/bram/jointomicscomp/data/sampleNames.npy")
cancertype = np.load("/home/bram/jointomicscomp/data/cancerType.npy")
cancertypes = np.load("/home/bram/jointomicscomp/data/cancerTypes.npy")
stageTypes = np.load("/home/bram/jointomicscomp/data/stageTypes.npy")<jupyter_output><empty_output><jupyter_text>Curated Data<jupyter_code>survival_table = pd.read_table("/home/bram/jointomicscomp/data/Survival_SupplementalTable_S1_20171025_xena_sp", index_col=0).loc[samples]

print("Shape:", survival_table.shape)<jupyter_output>Shape: (8396, 33)<jupyter_text>Show most common cancer types<jupyter_code>counter = dict()
cancer_types = survival_table.iloc[:, 1]
for value in cancer_types:
    if value in counter:
        counter[value] += 1
        
    else:
        counter[value] = 1
        
counter = sorted(counter.items(), key=lambda x: x[1], reverse=True)
counter
(unique, counts) = np.unique(cancertype, return_counts=True)
frequencies = np.asarray((unique, counts)).T
frequencies<jupyter_output><empty_output><jupyter_text>Select cancer type and data<jupyter_code># All patients with a certain cancer cancer.
ctype = 'LUSC'
ctype_idx = np.where(cancertypes == ctype)[0][0] # Get index of ctype in total cancers list
print(ctype_idx)

# Save all required data in data/cancer_type folder
save_dir = os.path.join("/home/bram/jointomicscomp/data", '{}'.format(ctype))
if not os.path.exists(save_dir):
    os.makedirs(save_dir)

cancer_type_info_table = survival_table.loc[survival_table["cancer type abbreviation"] == ctype]
print(cancer_type_info_table.shape)
cancer_type_patients = cancer_type_info_table["_PATIENT"]
    
cancer_type_patients<jupyter_output>16
(367, 33)<jupyter_text>Get RNA data from these patients grab corresponding stage<jupyter_code># Stages.
gene_expression_csv = pd.read_csv("/home/bram/jointomicscomp/data/rna-pancancer-5000-minmax.csv", index_col=0)
gene_expression_csv = gene_expression_csv.loc[samples]  # Take only samples used in this research
print("gene_expression_csv shape: ", gene_expression_csv.shape)

stages = dict()
s = set()
no_stage_counter = 0
no_stage_samples = []
for patient_idx, patient in enumerate(cancer_type_patients):
    # Get all occurences of the patient in the original data file.
    x = gene_expression_csv[[patient in label for label in samples]]
    
    y = survival_table[survival_table["_PATIENT"] == patient]["ajcc_pathologic_tumor_stage"].values[0]
    # Get strin to remove nans.
    y = str(y)
    s.add(y)
    if y == "nan" or y == 'Stage X' or y == '[Discrepancy]' or x.empty:
        no_stage_counter += 1
        print(x.index.values)
        no_stage_samples.append(x.index.values)
        continue
    
    # If it's stage 4 then add to stage for. No A's and B's here.
    if y == 'Stage IV':
        stage = 'Stage 4'
    
    # Create stage string for other stages than 4.
    else:
        stage = 'Stage ' + str(y.count('I'))
    
    # Create stage key.
    if stage not in stages:
        stages[stage] = []
    
    # Add to dict.
    stages[stage].append(x)
    
print("{} Samples did not have a stage in the info dataframe.".format(no_stage_counter))
    

no_stage_samples
no_stage_sample_indices = np.zeros(len(no_stage_samples), dtype=int)
for i, no_stage in enumerate(no_stage_samples):
    sample_idx = np.where(samples == no_stage)[0][0]
    no_stage_sample_indices[i] = sample_idx

    
# All samples of the cancer type that do not have a stage should be removed from the dataset
print("patient indices to be removed is : ", len(no_stage_sample_indices))
ge = np.delete(ge, no_stage_sample_indices, 0)
gcn = np.delete(gcn, no_stage_sample_indices, 0)
me = np.delete(me, no_stage_sample_indices, 0)
samples = np.delete(samples, no_stage_sample_indices)
cancertype = np.delete(cancertype, no_stage_sample_indices)

class Stage:
    
    def __init__(self, rna):
        self.patient_data = rna
# Add the dataframes together.
for stage in stageTypes:
    stages[stage] = Stage(pd.concat(stages[stage]))
    stages[stage].patient_data['Stage'] = stage

samples_ctr = 0
for x in stageTypes:
    print(x, stages[x].patient_data.shape)
    samples_ctr += len(stages[x].patient_data)
    
print("Total number of samples:", samples_ctr)
stages
# Use X to denote GE and Y for ME, take indices on Y later
X = ge

# variable stage contains cancer stage (indexed from stageTypes), y contains cancertypes
Xtype = X[cancertype == ctype_idx]
Xrest = X[cancertype != ctype_idx]

yrest = cancertype[cancertype != ctype_idx]

# All BRCA samples
Xtype_samples = samples[cancertype == ctype_idx]

print("Data contains {} samples of Xtype_samples type {}".format(Xtype_samples.shape[0], ctype))
np.save("{}/{}_sampleNames.npy".format(save_dir, ctype), Xtype_samples)
# Create stage array
stageType = np.zeros(Xtype_samples.shape[0])

for i in range(len(stageTypes)):
    stage_samples = stages[stageTypes[i]].patient_data.index.values
    print(stageTypes[i], stage_samples.shape[0])
    for stage_sample in stage_samples:
        idx = np.where(Xtype_samples == stage_sample)[0][0]
        stageType[idx] = i  # stages in StageTypes array are 0-indexed<jupyter_output>Stage 1 170
Stage 2 134
Stage 3 56
Stage 4 4<jupyter_text>Stratified Split Data<jupyter_code>split1 = StratifiedShuffleSplit(n_splits=1, test_size=0.1)

sss1 = split1.split(Xtype, stageType)

trainValidInd = 0
testInd = 0
for i, j in sss1:
    trainValidInd = i
    testInd = j
    


Xtest = Xtype[testInd]
print("Test samples with {}".format(ctype), Xtest.shape)
stagetest = stageType[testInd] 
np.save("{}/{}_test_stageType.npy".format(save_dir, ctype), stagetest)
np.save("{}/{}_test_samples.npy".format(save_dir, ctype), Xtype_samples[testInd])

XtrainValid = Xtype[trainValidInd]

stagetrainValid = stageType[trainValidInd]

split2 = StratifiedShuffleSplit(n_splits=1, test_size=1/9)

sss2 = split1.split(XtrainValid, stagetrainValid)

trainInd = 0
validInd = 0
for i, j in sss2:
    trainInd = i
    validInd = j

Xtrain = XtrainValid[trainInd]
print("Train samples with {}".format(ctype), Xtrain.shape)

stagetrain = stagetrainValid[trainInd]
np.save("{}/{}_train_stageType.npy".format(save_dir, ctype), stagetrain)
np.save("{}/{}_train_samples.npy".format(save_dir, ctype), Xtype_samples[trainValidInd][trainInd])


Xvalid = XtrainValid[validInd]
print("Validation samples with {}".format(ctype), Xvalid.shape)

stagevalid = stagetrainValid[validInd]
np.save("{}/{}_valid_stageType.npy".format(save_dir, ctype), stagevalid)
np.save("{}/{}_valid_samples.npy".format(save_dir, ctype), Xtype_samples[trainValidInd][validInd])


splitRest = StratifiedShuffleSplit(n_splits=1, test_size=1/9)

sss3 = splitRest.split(Xrest, yrest)
trainIndrest = 0
validIndrest = 0
for i, j in sss3:
    trainIndrest = i
    validIndrest = j


Xresttrain = Xrest[trainIndrest]
print("Train samples from remaining cancer types".format(ctype), Xresttrain.shape)


Xrestvalid = Xrest[validIndrest]
print("Validation samples from remaining cancer types".format(ctype), Xrestvalid.shape)


# Use same indices used to split X (= GE) on Y (= ME) and Z (= GCN)
Y = me
Ytype = Y[cancertype == ctype_idx]
Yrest = Y[cancertype != ctype_idx]


Ytest = Ytype[testInd]
YtrainValid = Ytype[trainValidInd]
Ytrain = YtrainValid[trainInd]
Yvalid = YtrainValid[validInd]

Yresttrain = Yrest[trainIndrest]
Yrestvalid = Yrest[validIndrest]

Z = gcn
Ztype = Z[cancertype == ctype_idx]
Zrest = Z[cancertype != ctype_idx]

Ztest = Ztype[testInd]
ZtrainValid = Ztype[trainValidInd]
Ztrain = ZtrainValid[trainInd]
Zvalid = ZtrainValid[validInd]

Zresttrain = Zrest[trainIndrest]
Zrestvalid = Zrest[validIndrest]

# Throw together BRCA training data and Rest training data (and same for validation)
XtrainAll = np.vstack((Xtrain, Xresttrain))

XvalidAll = np.vstack((Xvalid, Xrestvalid))

print(XtrainAll.shape)
print(XvalidAll.shape)
print(Xtest.shape)
(unique, counts) = np.unique(stagetest, return_counts=True)
frequencies = np.asarray((unique, counts)).T

print("List Cancer stage from {} and occurences in test set".format(ctype))
print(frequencies)
# Save all splits for testing convenience
np.save("{}/{}_GE_train".format(save_dir, ctype), Xtrain)
np.save("{}/rest_GE_train".format(save_dir), Xresttrain)
np.save("{}/{}_GE_valid".format(save_dir, ctype), Xvalid)
np.save("{}/rest_GE_valid".format(save_dir), Xrestvalid)
np.save("{}/{}_GE_test".format(save_dir, ctype), Xtest)

np.save("{}/{}_ME_train".format(save_dir, ctype), Ytrain)
np.save("{}/rest_ME_train".format(save_dir), Yresttrain)
np.save("{}/{}_ME_valid".format(save_dir, ctype), Yvalid)
np.save("{}/rest_ME_valid".format(save_dir), Yrestvalid)
np.save("{}/{}_ME_test".format(save_dir, ctype), Ytest)

np.save("{}/{}_GCN_train".format(save_dir, ctype), Ztrain)
np.save("{}/rest_GCN_train".format(save_dir), Zresttrain)
np.save("{}/{}_GCN_valid".format(save_dir, ctype), Zvalid)
np.save("{}/rest_GCN_valid".format(save_dir), Zrestvalid)
np.save("{}/{}_GCN_test".format(save_dir, ctype), Ztest)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13027640882794086, 'fraction_numerical': 0.009642168416541676, 'mean_word_length': 5.4114010989010985, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '248498', 'license': 'MIT', 'n_tokens_mistral': 3462, 'n_tokens_neox': 3306, 'n_words': 710, 'path': 'src/data_preprocessing/SurvivalPrediction_datasplit.ipynb', 'repo_name': 'brmprnk/jointomicscomp', 'size': '9334'}