<jupyter_start><jupyter_text>06 Region classifier**Objectives:*** train a classifier to predict Regions (`-->` multiclass)* explain classifications in different regions with LIME<jupyter_code>import pandas as pd
import numpy as np
import os
import pickle

import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns

from boruta import BorutaPy

from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor, GradientBoostingRegressor, GradientBoostingClassifier
from sklearn.model_selection import RandomizedSearchCV, cross_val_score, ShuffleSplit

import lime 
import sklearn.datasets
from lime.lime_tabular import LimeTabularExplainer

import rpy2.robjects as robjects
from rpy2.robjects import pandas2ri
pandas2ri.activate()

sns.set(style="whitegrid")<jupyter_output><empty_output><jupyter_text>--- Load the data<jupyter_code>wd = '/media/tmo/data/work/datasets/02_ST'

logcpm_path = wd + '/ashley_21.03.2018/logcpm_merge_20180212.pickle'
meta_path = wd + '/meta/meta.parquet'
%%time
meta_df = pd.read_parquet(meta_path)
%%time
logcpm_df = pickle.load(open(logcpm_path, "rb"))

logcpm_df.index.name = 'spot_UID'
logcpm_df.reset_index(inplace=True)
logcpm_df.rename(columns={'sampleID': 'slide_ID'}, inplace=True)
st_df = logcpm_df.merge(meta_df, how='inner', on=['spot_UID', 'slide_ID'])
st_df['slide_ID'] = st_df['slide_ID'].astype('category', copy=False)
st_df['GT'] = st_df['GT'].astype('category', copy=False)
st_df['age'] = st_df['age_GT'].astype('category', copy=False)
st_df['age_GT'] = st_df['age_GT'].astype('category', copy=False)
n_genes = 46454
gene_columns = st_df.columns[1:n_genes+1]
expression_df = st_df[gene_columns]
assert expression_df.shape == (10327, 46454)
region_levels_df = pd.read_csv('region_levels.csv', sep=',', keep_default_na=False)
genotype_df = st_df[['GT']]
region_df = st_df[['Region_predict']].copy()
region_df = region_df.merge(region_levels_df, on='Region_predict')
region_cat_df = region_df['Region_predict'].astype('category', copy=False)
region_level1_df = region_df['Level_01'].astype('category', copy=False)
region_level2_df = region_df['Level_02'].astype('category', copy=False)<jupyter_output><empty_output><jupyter_text>---<jupyter_code>region_df['Level_01'].value_counts()
region_df['Level_02'].value_counts()<jupyter_output><empty_output><jupyter_text>--- Extract *all-relevant* feature set: Region<jupyter_code>boruta_rf = RandomForestClassifier(n_jobs=-1, n_estimators=1000, max_features='sqrt', max_depth=5)

def train_feature_selector(X_df=expression_df,  # the transcriptome expression vectors
                           y_df=region_df,      # the Region column
                           estimator=boruta_rf, verbose=2, seed=42):  # boruta parameters
    feature_selector = BorutaPy(estimator=estimator, verbose=verbose, random_state=seed, n_estimators='auto')    
    
    X = X_df.as_matrix()
    y = y_df.values.ravel()
    feature_selector.fit(X, y)
    
    return feature_selector
Region_feature_selector = train_feature_selector(y_df = region_df)
Region_features = list(gene_columns[Region_feature_selector.support_])
pd.DataFrame(Region_features).to_csv('06_region_features.txt', index=None, header=None)<jupyter_output><empty_output><jupyter_text>Extract *all-relevant* feature set: Level 01<jupyter_code>level1_feature_selector = train_feature_selector(y_df = region_level1_df)
Level1_features = list(gene_columns[level1_feature_selector.support_])
pd.DataFrame(Level1_features).to_csv('06_level1_features.txt', index=None, header=None)<jupyter_output><empty_output><jupyter_text>Extract *all-relevant* feature set: Level 02<jupyter_code>level2_feature_selector = train_feature_selector(y_df = region_level2_df)
Level2_features = list(gene_columns[level2_feature_selector.support_])
pd.DataFrame(Level2_features).to_csv('06_level2_features.txt', index=None, header=None)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12253268392719815, 'fraction_numerical': 0.021020251217636503, 'mean_word_length': 7.9290617848970255, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '633592', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1451, 'n_tokens_neox': 1383, 'n_words': 215, 'path': 'notebooks/03_ashley_21_mar_2018/06_boruta_lime_Region.ipynb', 'repo_name': 'tmoerman/alois', 'size': '3901'}