<jupyter_start><jupyter_text>Load Dependencies and Raw Data<jupyter_code>import pandas as pd
import numpy as np
import dill, pickle
import copy

from collections import Counter
import itertools
from scipy import stats

import matplotlib.pyplot as plt

from sklearn.cluster import KMeans
from sklearn.cluster import AgglomerativeClustering
from sklearn.decomposition import FactorAnalysis

from skrebate import ReliefF, MultiSURF, MultiSURFstar
from sklearn.feature_selection import f_classif
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE

from sklearn import metrics
from sklearn.metrics import adjusted_rand_score, rand_score
from sklearn.metrics.cluster import pair_confusion_matrix

from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler
data = pd.read_excel('../data/GC-MS_data.xlsx')
# counts
ID_num = np.where(data.ID == 'Healthy', 0, 1)
data.insert(1, 'ID_num', ID_num)<jupyter_output><empty_output><jupyter_text>User Defined Functions<jupyter_code>def do_factor_analysis(dataset):
    fa = FactorAnalysis().fit(dataset)
    return fa.mean_, fa.get_covariance()
def bhatt_dist(m1,cov1,m2,cov2):
    cov = (1/2) * (cov1 + cov2)
    
    Term1 = (1/8) * (m1 - m2).T @ np.linalg.inv(cov) @ (m1 - m2)
    Term2 = (1 / 2) * np.log(np.linalg.det(cov) / np.sqrt(np.linalg.det(cov1) * np.linalg.det(cov2)))
    
    return Term1+Term2, Term1, Term2<jupyter_output><empty_output><jupyter_text>Full Dataset We need a metric that is better than randIndex. randIndex is label agnostic. in otherwords, if 2 instances for the negative class are clustered together it is a positive outcome for randIndex even if they are clustered in the same cluster as the positive instances. Need to try out log transorfm of the data<jupyter_code>data.head()
Counter(data.ID)
data.shape<jupyter_output><empty_output><jupyter_text>Scale the data<jupyter_code>X_raw_all = data.values[:,2:]
X_scaled_all = StandardScaler().fit_transform(X_raw_all)
data_scaled_all = pd.DataFrame(X_scaled_all, columns = data.columns[2:])
data_scaled_all.insert(0, 'ID', data.ID.values)
data_scaled_all.insert(1, 'ID_num', data.ID_num.values)
data_scaled_all.head()
data_healthy_all_df = data_scaled_all.loc[data_scaled_all.ID == 'Healthy']
data_asthma_all_df = data_scaled_all.loc[data_scaled_all.ID == 'Asthmatic']
data_healthy_all = data_healthy_all_df.values[:,2:]
data_asthma_all = data_asthma_all_df.values[:, 2:]
healthy_mean, healthy_cov = do_factor_analysis(data_healthy_all)
asthma_mean, asthma_cov = do_factor_analysis(data_asthma_all)
dist, t1, t2 = bhatt_dist(healthy_mean, healthy_cov, asthma_mean, asthma_cov)
print(dist)
print(np.linalg.det(healthy_cov))
print(np.linalg.det(asthma_cov))<jupyter_output>/Users/boses1/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:5: RuntimeWarning: invalid value encountered in double_scalars
  """<jupyter_text>Relief Methods<jupyter_code>data_scaled_df = data_scaled_all
data_scaled_df.head()<jupyter_output><empty_output><jupyter_text>Relief-F<jupyter_code>fs = ReliefF(discrete_threshold = 5, n_jobs=1)
fs.fit(data_scaled_df.values[:,2:].astype(float), data_scaled_df.ID_num.values)

feature_scores = fs.feature_importances_
feature_ids = np.where(feature_scores>=0)[0]
selected_features = np.array(data_scaled_df.columns[2:][feature_ids]) 
X_reliefF = data_scaled_df.values[:,2:][:,feature_ids]
X_reliefF.shape
X_reliefF_df = pd.DataFrame(X_reliefF, columns = selected_features)
X_reliefF_df.insert(0, 'ID', data.ID.values)
X_reliefF_df.head()
data_healthy_df = X_reliefF_df.loc[X_reliefF_df.ID == 'Healthy']
data_asthma_df = X_reliefF_df.loc[X_reliefF_df.ID == 'Asthmatic']
data_healthy = data_healthy_df.values[:,1:]
data_asthma = data_asthma_df.values[:, 1:]
healthy_mean, healthy_cov = do_factor_analysis(data_healthy)
asthma_mean, asthma_cov = do_factor_analysis(data_asthma)
dist, t1, t2 = bhatt_dist(healthy_mean, healthy_cov, asthma_mean, asthma_cov)
print(dist)
print(np.linalg.det(healthy_cov))
print(np.linalg.det(asthma_cov))<jupyter_output>nan
0.0
0.0<jupyter_text>MultiSURF<jupyter_code>fs = MultiSURF(discrete_threshold = 5, n_jobs=1)
fs.fit(data_scaled_df.values[:,2:].astype(float), data_scaled_df.ID_num.values)

feature_scores = fs.feature_importances_
feature_ids = np.where(feature_scores>=0)[0]
selected_features = np.array(data_scaled_df.columns[2:][feature_ids]) 
X_MultiSURF = data_scaled_df.values[:,2:][:,feature_ids]
X_MultiSURF.shape
X_MultiSURF_df = pd.DataFrame(X_MultiSURF, columns = selected_features)
X_MultiSURF_df.insert(0, 'ID', data.ID.values)
X_MultiSURF_df.head()
data_healthy_df = X_MultiSURF_df.loc[X_reliefF_df.ID == 'Healthy']
data_asthma_df = X_MultiSURF_df.loc[X_reliefF_df.ID == 'Asthmatic']
data_healthy = data_healthy_df.values[:,1:]
data_asthma = data_asthma_df.values[:, 1:]
healthy_mean, healthy_cov = do_factor_analysis(data_healthy)
asthma_mean, asthma_cov = do_factor_analysis(data_asthma)
dist, t1, t2 = bhatt_dist(healthy_mean, healthy_cov, asthma_mean, asthma_cov)
print(dist)
print(np.linalg.det(healthy_cov))
print(np.linalg.det(asthma_cov))<jupyter_output>nan
0.0
0.0<jupyter_text>MultiSURFStar<jupyter_code>fs = MultiSURFstar(discrete_threshold = 5, n_jobs=1)
fs.fit(data_scaled_df.values[:,2:].astype(float), data_scaled_df.ID_num.values)

feature_scores = fs.feature_importances_
feature_ids = np.where(feature_scores>=0)[0]
selected_features = np.array(data_scaled_df.columns[2:][feature_ids]) 
X_MultiSURFStar = data_scaled_df.values[:,2:][:,feature_ids]
X_MultiSURFStar.shape
X_MultiSURFStar_df = pd.DataFrame(X_MultiSURFStar, columns = selected_features)
X_MultiSURFStar_df.insert(0, 'ID', data.ID.values)
X_MultiSURFStar_df.head()
data_healthy_df = X_MultiSURFStar_df.loc[X_MultiSURFStar_df.ID == 'Healthy']
data_asthma_df = X_MultiSURFStar_df.loc[X_MultiSURFStar_df.ID == 'Asthmatic']
data_healthy = data_healthy_df.values[:,1:]
data_asthma = data_asthma_df.values[:, 1:]
healthy_mean, healthy_cov = do_factor_analysis(data_healthy)
asthma_mean, asthma_cov = do_factor_analysis(data_asthma)
dist, t1, t2 = bhatt_dist(healthy_mean, healthy_cov, asthma_mean, asthma_cov)
print(dist)
print(np.linalg.det(healthy_cov))
print(np.linalg.det(asthma_cov))<jupyter_output>0.8781896505522983
0.10470358387067949
0.0049995991493347255<jupyter_text>Univariate Statistical Feature Selection  Anova<jupyter_code>f,p = f_classif(data_scaled_df.values[:,2:].astype(float), data.ID_num.values)
feature_ids = np.where(p<=0.05)[0]


selected_features = np.array(data_scaled_df.columns[2:][feature_ids]) 

X_anova = data_scaled_df.values[:,2:][:,feature_ids]
X_anova.shape
X_anova_df = pd.DataFrame(X_anova, columns = selected_features)
X_anova_df.insert(0, 'ID', data.ID.values)
X_anova_df.head()
data_healthy_df = X_anova_df.loc[X_anova_df.ID == 'Healthy']
data_asthma_df = X_anova_df.loc[X_anova_df.ID == 'Asthmatic']
data_healthy = data_healthy_df.values[:,1:]
data_asthma = data_asthma_df.values[:, 1:]
healthy_mean, healthy_cov = do_factor_analysis(data_healthy)
asthma_mean, asthma_cov = do_factor_analysis(data_asthma)
dist, t1, t2 = bhatt_dist(healthy_mean, healthy_cov, asthma_mean, asthma_cov)
print(dist)
print(np.linalg.det(healthy_cov))
print(np.linalg.det(asthma_cov))<jupyter_output>inf
1.2922052962755676e-110
2.6274739360408576e-256<jupyter_text>Combinations  Anova + Relief-F<jupyter_code>X_anova.shape
fs = ReliefF(discrete_threshold = 5, n_jobs=1)
fs.fit(X_anova.astype(float), data.ID_num.values)

feature_scores = fs.feature_importances_
feature_ids = np.where(feature_scores>=0)[0]
selected_features = selected_features[feature_ids]
X_ano_reliefF = X_anova[:,feature_ids]
X_ano_reliefF.shape
X_anova_relief_df = pd.DataFrame(X_ano_reliefF, columns = selected_features)
X_anova_relief_df.insert(0, 'ID', data.ID.values)
X_anova_relief_df.head()
data_healthy_df = X_anova_relief_df.loc[X_anova_relief_df.ID == 'Healthy']
data_asthma_df = X_anova_relief_df.loc[X_anova_relief_df.ID == 'Asthmatic']
data_healthy = data_healthy_df.values[:,1:]
data_asthma = data_asthma_df.values[:, 1:]
healthy_mean, healthy_cov = do_factor_analysis(data_healthy)
asthma_mean, asthma_cov = do_factor_analysis(data_asthma)
dist, t1, t2 = bhatt_dist(healthy_mean, healthy_cov, asthma_mean, asthma_cov)
print(dist)
print(np.linalg.det(healthy_cov))
print(np.linalg.det(asthma_cov))<jupyter_output>87.19351047162358
2.3008692548762507e-13
6.307301839056216e-162<jupyter_text>Dataset with linearly correlated features removed<jupyter_code>with open('../data/independent_features.pik', "rb") as f:
    independent_features = dill.load(f)
X_no_corr_df = independent_features['X_no_corr_df']
X_no_corr_df.shape
X_no_corr_df.head()
X_scaled = StandardScaler().fit_transform(X_no_corr_df.values)
data_scaled_df = pd.DataFrame(X_scaled, columns = X_no_corr_df.columns)
data_scaled_df.insert(0, 'ID', data.ID.values)
data_scaled_df.insert(1, 'ID_num', data.ID_num.values)
data_scaled_df.head()
data_healthy_df = data_scaled_df.loc[data_scaled_df.ID == 'Healthy']
data_asthma_df = data_scaled_df.loc[data_scaled_df.ID == 'Asthmatic']
data_healthy = data_healthy_df.values[:,2:]
data_asthma = data_asthma_df.values[:, 2:]
healthy_mean, healthy_cov = do_factor_analysis(data_healthy)
asthma_mean, asthma_cov = do_factor_analysis(data_asthma)
dist, t1, t2 = bhatt_dist(healthy_mean, healthy_cov, asthma_mean, asthma_cov)
dist<jupyter_output><empty_output><jupyter_text>Relief Methods<jupyter_code>data_scaled_df.head()<jupyter_output><empty_output><jupyter_text>Relief-F<jupyter_code>fs = ReliefF(discrete_threshold = 5, n_jobs=1)
fs.fit(data_scaled_df.values[:,2:].astype(float), data_scaled_df.ID_num.values)

feature_scores = fs.feature_importances_
feature_ids = np.where(feature_scores>=0)[0]
selected_features = np.array(X_no_corr_df.columns[feature_ids]) 
X_reliefF = data_scaled_df.values[:,2:][:,feature_ids]
X_reliefF.shape
X_reliefF_df = pd.DataFrame(X_reliefF, columns = selected_features)
X_reliefF_df.insert(0, 'ID', data.ID.values)
X_reliefF_df.head()
data_healthy_df = X_reliefF_df.loc[X_reliefF_df.ID == 'Healthy']
data_asthma_df = X_reliefF_df.loc[X_reliefF_df.ID == 'Asthmatic']
data_healthy = data_healthy_df.values[:,1:]
data_asthma = data_asthma_df.values[:, 1:]
healthy_mean, healthy_cov = do_factor_analysis(data_healthy)
asthma_mean, asthma_cov = do_factor_analysis(data_asthma)
dist, t1, t2 = bhatt_dist(healthy_mean, healthy_cov, asthma_mean, asthma_cov)
np.linalg.det(healthy_cov)<jupyter_output><empty_output><jupyter_text>MultiSURF<jupyter_code>fs = MultiSURF(discrete_threshold = 5, n_jobs=1)
fs.fit(data_scaled_df.values[:,2:].astype(float), data_scaled_df.ID_num.values)

feature_scores = fs.feature_importances_
feature_ids = np.where(feature_scores>=0)[0]
selected_features = np.array(X_no_corr_df.columns[feature_ids]) 
X_MultiSURF = data_scaled_df.values[:,2:][:,feature_ids]
X_MultiSURF.shape
X_MultiSURF_df = pd.DataFrame(X_MultiSURF, columns = selected_features)
X_MultiSURF_df.insert(0, 'ID', data.ID.values)
X_MultiSURF_df.head()
data_healthy_df = X_MultiSURF_df.loc[X_reliefF_df.ID == 'Healthy']
data_asthma_df = X_MultiSURF_df.loc[X_reliefF_df.ID == 'Asthmatic']
data_healthy = data_healthy_df.values[:,1:]
data_asthma = data_asthma_df.values[:, 1:]
healthy_mean, healthy_cov = do_factor_analysis(data_healthy)
asthma_mean, asthma_cov = do_factor_analysis(data_asthma)
dist, t1, t2 = bhatt_dist(healthy_mean, healthy_cov, asthma_mean, asthma_cov)
dist<jupyter_output><empty_output><jupyter_text>MultiSURFStar<jupyter_code>fs = MultiSURFstar(discrete_threshold = 5, n_jobs=1)
fs.fit(data_scaled_df.values[:,2:].astype(float), data_scaled_df.ID_num.values)

feature_scores = fs.feature_importances_
feature_ids = np.where(feature_scores>=0)[0]
selected_features = np.array(X_no_corr_df.columns[feature_ids]) 
X_MultiSURFStar = data_scaled_df.values[:,2:][:,feature_ids]
X_MultiSURFStar.shape
X_MultiSURFStar_df = pd.DataFrame(X_MultiSURFStar, columns = selected_features)
X_MultiSURFStar_df.insert(0, 'ID', data.ID.values)
X_MultiSURFStar_df.head()
data_healthy_df = X_MultiSURFStar_df.loc[X_MultiSURFStar_df.ID == 'Healthy']
data_asthma_df = X_MultiSURFStar_df.loc[X_MultiSURFStar_df.ID == 'Asthmatic']
data_healthy = data_healthy_df.values[:,1:]
data_asthma = data_asthma_df.values[:, 1:]
healthy_mean, healthy_cov = do_factor_analysis(data_healthy)
asthma_mean, asthma_cov = do_factor_analysis(data_asthma)
dist, t1, t2 = bhatt_dist(healthy_mean, healthy_cov, asthma_mean, asthma_cov)
dist<jupyter_output><empty_output><jupyter_text>Univariate Statistical Feature Selection  Anova<jupyter_code>f,p = f_classif(data_scaled_df.values[:,2:].astype(float), data.ID_num.values)
feature_ids = np.where(p<=0.05)[0]


selected_features = np.array(X_no_corr_df.columns[feature_ids])

X_anova = data_scaled_df.values[:,2:][:,feature_ids]
X_anova.shape
X_anova_df = pd.DataFrame(X_anova, columns = selected_features)
X_anova_df.insert(0, 'ID', data.ID.values)
X_anova_df.head()
data_healthy_df = X_anova_df.loc[X_anova_df.ID == 'Healthy']
data_asthma_df = X_anova_df.loc[X_anova_df.ID == 'Asthmatic']
data_healthy = data_healthy_df.values[:,1:]
data_asthma = data_asthma_df.values[:, 1:]
healthy_mean, healthy_cov = do_factor_analysis(data_healthy)
asthma_mean, asthma_cov = do_factor_analysis(data_asthma)
dist, t1, t2 = bhatt_dist(healthy_mean, healthy_cov, asthma_mean, asthma_cov)
dist
np.linalg.det(healthy_cov)
np.linalg.det(asthma_cov)<jupyter_output><empty_output><jupyter_text>Combinations  Anova + Relief-F<jupyter_code>X_anova.shape
fs = ReliefF(discrete_threshold = 5, n_jobs=1)
fs.fit(X_anova.astype(float), data.ID_num.values)

feature_scores = fs.feature_importances_
feature_ids = np.where(feature_scores>=0)[0]
selected_features = selected_features[feature_ids]
X_ano_reliefF = X_anova[:,feature_ids]
X_ano_reliefF.shape
X_anova_relief_df = pd.DataFrame(X_ano_reliefF, columns = selected_features)
X_anova_relief_df.insert(0, 'ID', data.ID.values)
X_anova_relief_df.head()
data_healthy_df = X_anova_relief_df.loc[X_anova_relief_df.ID == 'Healthy']
data_asthma_df = X_anova_relief_df.loc[X_anova_relief_df.ID == 'Asthmatic']
data_healthy = data_healthy_df.values[:,1:]
data_asthma = data_asthma_df.values[:, 1:]
data_scaled_df.head()
import random
random_feat_ids = list(range(681))
random.shuffle(random_feat_ids)
data_scaled_df.head()
data_healthy = data_scaled_df.loc[data_scaled_df.ID == 'Healthy'].values[:,2:]
data_asthma = data_scaled_df.loc[data_scaled_df.ID == 'Asthmatic'].values[:,2:]

healthy_mean, healthy_cov = do_factor_analysis(data_healthy[:,random_feat_ids[:30]])
asthma_mean, asthma_cov = do_factor_analysis(data_asthma[:,random_feat_ids[:30]])

dist, t1, t2 = bhatt_dist(healthy_mean, healthy_cov, asthma_mean, asthma_cov)
print(dist)
print(np.linalg.det(healthy_cov))
print(np.linalg.det(asthma_cov))
t2
np.linalg.det(asthma_cov)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12176549865229111, 'fraction_numerical': 0.02183288409703504, 'mean_word_length': 9.095918367346938, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '147145', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 6430, 'n_tokens_neox': 6056, 'n_words': 799, 'path': 'notebooks/.ipynb_checkpoints/FactorAnalysis-checkpoint.ipynb', 'repo_name': 'sauravbose/asthma-biomarker', 'size': '14840'}