<jupyter_start><jupyter_text>Baseline methods<jupyter_code>import src
import matplotlib.pyplot as plt
import seaborn as sns
import os
from functools import partial

import numpy as np
from sklearn.decomposition import PCA

from sklearn.dummy import DummyClassifier
from sklearn.svm import LinearSVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier

from sklearn.metrics import confusion_matrix
from sklearn.model_selection import StratifiedKFold
from sklearn.pipeline import Pipeline

from sklearn.base import BaseEstimator
from sklearn.feature_selection.base import SelectorMixin
from sklearn.preprocessing import MinMaxScaler
from sklearn.utils.validation import check_is_fitted

import tensorflow as tf
from tensorflow.keras import backend as K
from tensorflow.keras.layers import Dense
from tensorflow.keras.models import Sequential
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.wrappers.scikit_learn import KerasClassifier


K.set_session(tf.Session(config=tf.ConfigProto(intra_op_parallelism_threads=1, 
                                               inter_op_parallelism_threads=1, 
                                               allow_soft_placement=True, 
                                               device_count = {'CPU': 10})))
tissues = {
    'breast': ['BRCA'],
    'lung': ["LUSC", "LUAD"],
    'kidney': ["KIRC", "KICH", "KIRP"],
    'bladder': ['BLCA']
}
cohorts = {}

idx_to_gene, _ = src.data.load_gene_indices()
for tissue, datasets in tissues.items():
    print(tissue)
    X_tissue, idx_to_sample_tissue, y_tissue = np.empty((0, idx_to_gene.shape[0])), [], np.empty(0, dtype=int)
    for d in datasets:
        X_d, idx_to_sample_d, _, y_d, classes = src.data.get_normal_vs_tumor_task(d)
        X_tissue = np.vstack([X_tissue, X_d])
        idx_to_sample_tissue = np.append(idx_to_sample_tissue, idx_to_sample_d.values)
        y_tissue = np.append(y_tissue, y_d)
    idx_to_sample_tissue = pd.Series(data=idx_to_sample_tissue, index=np.arange(X_tissue.shape[0], dtype=int))
    cohorts[tissue] = (X_tissue, idx_to_sample_tissue, idx_to_gene, y_tissue, classes)<jupyter_output>breast
lung
kidney
bladder<jupyter_text>PCA analysis<jupyter_code>color_mapping = {
    'Solid Tissue Normal': '#191970',
    'Primary Tumor': "#FF3232"
}

def plot_PCA(X, y, classes, color_mapping):
    pca = PCA(n_components=2)
    X_pca = pca.fit_transform(X)
    for cy in np.unique(y):
        sns.regplot(X_pca[y == cy, 0], X_pca[y == cy, 1], color=color_mapping[classes[cy]], fit_reg=False, label=classes[cy])
    plt.legend()
    plt.xlabel("1st component ({:.2f}%)".format(pca.explained_variance_ratio_[0] * 100))
    plt.ylabel("2nd component ({:.2f}%)".format(pca.explained_variance_ratio_[1] * 100))
plt.figure(figsize=(30, 5))
i = 1
for cohort in cohorts.keys():
    X_cohort, _, _, y_cohort, classes_cohort = cohorts[cohort]
    plt.subplot(1, len(cohorts), i)
    plot_PCA(X_cohort, y_cohort, classes_cohort, color_mapping)
    plt.title(cohort)
    i += 1
plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Baseline classifiers  Top Variant Feature Selection<jupyter_code>class TopVariantSelector(BaseEstimator, SelectorMixin):
    """ A very simple feature selector which uses the top variant features """
    def __init__(self, top_k):
        self.top_k = top_k
    
    def fit(self, X, y=None):
        stds = X.std(0) # 1 x n.genes
        selected_genes = np.argsort(stds)[::-1][:self.top_k]
        self.selected_features_ = selected_genes
        self.mask_ = np.in1d(np.arange(X.shape[1]), selected_genes)
        return self
        
    def _get_support_mask(self):
        check_is_fitted(self, 'selected_features_')
        check_is_fitted(self, 'mask_')
        return self.mask_<jupyter_output><empty_output><jupyter_text>Metrics<jupyter_code>def get_confusion_matrix_old(y_true, y_pred, names):
    return pd.DataFrame(data=confusion_matrix(y_true=y_true, y_pred=y_pred), 
                        index=pd.Series(names, name='True'), columns=pd.Series(names, name='Predicted'))

def get_confusion_matrix(y_true, y_pred):
    normal_normal, normal_tumor, tumor_normal, tumor_tumor = confusion_matrix(y_true=y_true, y_pred=y_pred).flatten()
    cm = pd.DataFrame(data=[[normal_normal, normal_tumor, tumor_normal, tumor_tumor]], 
                      columns=['tp', 'fn', 'fp', 'tn'])
    cm.index.name = 'true_observed'
    return cm

def sum_confusion_matrices(results_cohort):
    r_cm = None
    for _, cm in results_cohort.items():
        if r_cm is None:
            r_cm = cm
        else:
            r_cm += cm
    return cm
def tumor_alone_model(input_size):
    """ A super-simple NN for the single tumor classification
    """
    model = Sequential()
    model.add(Dense(100, activation='relu'))
    model.add(Dense(10, activation='relu'))
    model.add(Dense(1, activation="sigmoid"))
    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
    return model
seed = 42
n_repeated = 5
n_folds = 5
top_variant = 5000

preprocessing = [('variance_selector', TopVariantSelector(top_k=top_variant)), ('minmax_scaler', MinMaxScaler())]

pipelines = {
    'most_frequent': Pipeline(preprocessing + [('classifier', DummyClassifier(strategy='most_frequent', random_state=seed))]),
    'SVM': Pipeline(preprocessing + [('classifier', LinearSVC(max_iter=10000, random_state=seed))]),
    'KNN': Pipeline(preprocessing + [('classifier', KNeighborsClassifier(n_neighbors=5))]),
    'RF': Pipeline(preprocessing + [('classifier', RandomForestClassifier(n_estimators=10, random_state=seed))]),
    'FFNN': Pipeline(preprocessing + [('classifier', KerasClassifier(partial(tumor_alone_model, input_size=top_variant), 
                                                                     epochs=20, batch_size=10, verbose=0, validation_split=0.3,
                                                                     callbacks = [EarlyStopping(monitor='val_acc', min_delta=0, 
                                                                                                patience=0, verbose=0,
                                                                                                restore_best_weights=True)]))])
}

def repeated_cross_validation(pipelines, X_tissue, y_tissue, seed=42, n_repeated=5, n_folds=5):
    results = {}

    for t in range(n_repeated):
        print("Repetition {}".format(t + 1))
        skf = StratifiedKFold(n_splits=n_folds, random_state=seed + t, shuffle=True)
        i = 0
        for train_idx, test_idx in skf.split(X_tissue, y_tissue):
            X_train, y_train = X_tissue[train_idx, :], y_tissue[train_idx]
            X_test, y_test = X_tissue[test_idx, :], y_tissue[test_idx]
            print("\tSplit {}\tTrain ({}, {})\tTest ({}, {})".format(i, y_train[y_train == 0].shape[0], 
                                                                     y_train[y_train == 1].shape[0],
                                                                     y_test[y_test == 0].shape[0], 
                                                                     y_test[y_test == 1].shape[0]))
            for pname, p in pipelines.items():
                print("\t\tPipeline {}".format(pname))
                p.fit(X_train, y_train)
                y_pred = p.predict(X_test)
                cm_tip = get_confusion_matrix(y_true=y_test, y_pred=y_pred)
                cm_tip['repetition'] = t
                cm_tip['fold'] = i

                if pname in results.keys():
                    results[pname] = pd.concat([results[pname], cm_tip], axis=0, ignore_index=True)
                    results[pname].index.name = 'true_predicted'
                else:
                    results[pname] = cm_tip
            i += 1
    return results
for tissue in cohorts.keys():
    X_tissue, _, _, y_tissue, classes = cohorts[tissue]
    os.makedirs(src.reports_dir / "confusion_matrices" / tissue, exist_ok=True)
    print("Tissue {}".format(tissue))
    results = repeated_cross_validation(pipelines, X_tissue, y_tissue, seed=seed, n_repeated=n_repeated, n_folds=n_folds)
    for pname, presults in results.items():
        presults.to_csv(src.reports_dir / "confusion_matrices" / tissue / "{}.tsv".format(pname), 
                        sep="\t", index=True, header=True, index_label="true_predicted")<jupyter_output>Tissue bladder
Repetition 1
	Split 0	Train (15, 325)	Test (4, 82)
		Pipeline most_frequent
		Pipeline SVM
		Pipeline KNN
		Pipeline RF
		Pipeline FFNN
	Split 1	Train (15, 325)	Test (4, 82)
		Pipeline most_frequent
		Pipeline SVM
		Pipeline KNN
		Pipeline RF
		Pipeline FFNN
	Split 2	Train (15, 326)	Test (4, 81)
		Pipeline most_frequent
		Pipeline SVM
		Pipeline KNN
		Pipeline RF
		Pipeline FFNN
	Split 3	Train (15, 326)	Test (4, 81)
		Pipeline most_frequent
		Pipeline SVM
		Pipeline KNN
		Pipeline RF
		Pipeline FFNN
	Split 4	Train (16, 326)	Test (3, 81)
		Pipeline most_frequent
		Pipeline SVM
		Pipeline KNN
		Pipeline RF
		Pipeline FFNN
Repetition 2
	Split 0	Train (15, 325)	Test (4, 82)
		Pipeline most_frequent
		Pipeline SVM
		Pipeline KNN
		Pipeline RF
		Pipeline FFNN
	Split 1	Train (15, 325)	Test (4, 82)
		Pipeline most_frequent
		Pipeline SVM
		Pipeline KNN
		Pipeline RF
		Pipeline FFNN
	Split 2	Train (15, 326)	Test (4, 81)
		Pipeline most_frequent
		Pipeline SVM
		Pipeline KNN
		Pip[...]
{'fraction_non_alphanumeric': 0.11010926063434814, 'fraction_numerical': 0.01760899543863371, 'mean_word_length': 2.9055509527754766, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 29, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '597444', 'license': 'MIT', 'n_tokens_mistral': 3360, 'n_tokens_neox': 3050, 'n_words': 658, 'path': 'notebooks/Classical_ML.ipynb', 'repo_name': 'DEIB-GECO/CIBB2018', 'size': '9427'}