<jupyter_start><jupyter_text>Framing as continuous problem<jupyter_code>continuous_results_columns = ['method', 'covar selection', 'covars', 'mean_absolute_error', 'binarised_auc', 'binarised_accuracy']
continuous_results_df = pd.DataFrame(columns=continuous_results_columns)
def evaluate_continuous_model(model, model_name=None):
    if model_name is None:
        model_name = model.__class__.__name__
    volumetric_cv_results = cross_validate(model, volumetric_covars, outcome, cv=5, scoring=('neg_mean_absolute_error', 'neg_mean_absolute_percentage_error'))
    volumetric_cv_predictions = cross_val_predict(model, volumetric_covars, outcome, cv=5)
    volumetric_binarised_cv_predictions = volumetric_cv_predictions < (4.5 * 60)

    volumetric_result_df = pd.DataFrame([[model_name, 'volumetric', volumetric_covars_names,
                                  abs(volumetric_cv_results['test_neg_mean_absolute_error']).mean(),
                                  metrics.roc_auc_score(binarised_outcome, volumetric_binarised_cv_predictions),
                                  metrics.accuracy_score(binarised_outcome, volumetric_binarised_cv_predictions)
                                  ]], columns=continuous_results_columns)

    all_covar_cv_results = cross_validate(model, all_covars, outcome, cv=5, scoring=('neg_mean_absolute_error', 'neg_mean_absolute_percentage_error'))
    all_covar_cv_predictions = cross_val_predict(model, all_covars, outcome, cv=5)
    all_covar_binarised_cv_predictions = all_covar_cv_predictions < (4.5 * 60)

    all_covar_cv_result_df = pd.DataFrame([[model_name, 'volumetric + clinical',
                                            volumetric_covars_names + clinical_covars_names,
                                  abs(all_covar_cv_results['test_neg_mean_absolute_error']).mean(),
                                  metrics.roc_auc_score(binarised_outcome, all_covar_binarised_cv_predictions),
                                  metrics.accuracy_score(binarised_outcome, all_covar_binarised_cv_predictions)
                                  ]], columns=continuous_results_columns)

    return pd.concat([volumetric_result_df, all_covar_cv_result_df])

from sklearn.feature_selection import RFECV

linreg = LinearRegression()
lin_results_df = evaluate_continuous_model(linreg, LinearRegression().__class__.__name__)
continuous_results_df = continuous_results_df.append(lin_results_df)
lin_results_df

from sklearn.neural_network import MLPRegressor

mlp = MLPRegressor(random_state=1, max_iter=1000)

mlp_results_df = evaluate_continuous_model(mlp)

continuous_results_df = continuous_results_df.append(mlp_results_df)

mlp_results_df
from sklearn import linear_model

clf = linear_model.Lasso()

clf_results_df = evaluate_continuous_model(clf)
continuous_results_df = continuous_results_df.append(clf_results_df)

clf_results_df

svr = make_pipeline(preprocessing.StandardScaler(), svm.SVR())

svr_results_df = evaluate_continuous_model(svr, svm.SVR().__class__.__name__)
continuous_results_df = continuous_results_df.append(svr_results_df)

svr_results_df
from sklearn.tree import DecisionTreeRegressor

DTregressor = DecisionTreeRegressor()

dtr_results = evaluate_continuous_model(DTregressor)
continuous_results_df = continuous_results_df.append(dtr_results)

dtr_results<jupyter_output><empty_output><jupyter_text>Framing as binary problem<jupyter_code>binary_results_columns = ['method', 'covar selection', 'covars', 'auc', 'accuracy', 'f1', 'precision', 'recall']
binary_results_df = pd.DataFrame(columns=binary_results_columns)

def evaluate_binary_model(model, model_name=None):
    if model_name is None:
        model_name = model.__class__.__name__
    scoring = ('roc_auc', 'accuracy', 'f1', 'precision', 'recall')
    volumetric_cv_results = cross_validate(model, volumetric_covars, binarised_outcome, cv=5, scoring=scoring)

    volumetric_result_df = pd.DataFrame([[model_name, 'volumetric', volumetric_covars_names,
                                  np.median(volumetric_cv_results['test_roc_auc']),
                                  np.median(volumetric_cv_results['test_accuracy']),
                                  np.median(volumetric_cv_results['test_f1']),
                                  np.median(volumetric_cv_results['test_precision']),
                                  np.median(volumetric_cv_results['test_recall']),
                                  ]], columns=binary_results_columns)

    all_covar_cv_results = cross_validate(model, all_covars, binarised_outcome, cv=5, scoring=scoring)

    all_covar_cv_result_df = pd.DataFrame([[model_name, 'volumetric + clinical',
                                            volumetric_covars_names + clinical_covars_names,
                                  np.median(all_covar_cv_results['test_roc_auc']),
                                  np.median(all_covar_cv_results['test_accuracy']),
                                  np.median(all_covar_cv_results['test_f1']),
                                  np.median(all_covar_cv_results['test_precision']),
                                  np.median(all_covar_cv_results['test_recall']),
                                  ]], columns=binary_results_columns)

    return pd.concat([volumetric_result_df, all_covar_cv_result_df])
list(metrics.SCORERS.keys())
logreg = LogisticRegression()

logreg_results_df = evaluate_binary_model(logreg)
binary_results_df = binary_results_df.append(logreg_results_df)
logreg_results_df

from sklearn.ensemble import RandomForestClassifier

rfm = RandomForestClassifier()
rfm_results_df = evaluate_binary_model(rfm)
binary_results_df = binary_results_df.append(rfm_results_df)
rfm_results_df
clf_svm = svm.SVC()
clf_svm_results_df = evaluate_binary_model(clf_svm)
binary_results_df = binary_results_df.append(clf_svm_results_df)
clf_svm_results_df
from sklearn.model_selection import RandomizedSearchCV
from sklearn.neural_network import MLPClassifier

tuned_parameters = {
        'activation': (['relu','logistic']),
        'hidden_layer_sizes':([[80,160,80],[78,156,78],[88,176,88],[80,160]]),
        'alpha':     ([0.01, 0.001, 0.0001]),
        'batch_size':         [32,64],
        'learning_rate_init':    [0.01, 0.001],
        'solver': ["adam"]}

mlp = MLPClassifier(max_iter=5000)
mlp_clf = RandomizedSearchCV(mlp, tuned_parameters)

mlp_svm_results_df = evaluate_binary_model(mlp_clf)
binary_results_df = binary_results_df.append(mlp_svm_results_df)
mlp_svm_results_df
always_true = np.ones(len(binarised_outcome))
always_true_results = pd.DataFrame([['always_true', 'none', [],
                                  metrics.roc_auc_score(binarised_outcome, always_true),
                                  metrics.accuracy_score(binarised_outcome, always_true),
                                  metrics.f1_score(binarised_outcome, always_true),
                                  metrics.precision_score(binarised_outcome, always_true),
                                  metrics.recall_score(binarised_outcome, always_true),
                                  ]], columns=binary_results_columns)

binary_results_df = binary_results_df.append(always_true_results)

always_true_results
import os
output_dir = '/Users/jk1/OneDrive - unige.ch/stroke_research/scope/onset_output/volumetric_performance'
binary_results_df.to_excel(os.path.join(output_dir, 'binary_prediction_from_volumetric_variables.xlsx'))
continuous_results_df.to_excel(os.path.join(output_dir, 'continuous_prediction_from_volumetric_variables.xlsx'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08943951238902875, 'fraction_numerical': 0.010335232542732211, 'mean_word_length': 3.738229755178908, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '526767', 'license': 'MIT', 'n_tokens_mistral': 2512, 'n_tokens_neox': 2350, 'n_words': 287, 'path': 'scope/volumentric_approach/volumetric_prediction_exploration.ipynb', 'repo_name': 'JulianKlug/scop', 'size': '7547'}