<jupyter_start><jupyter_text>Learning Curves for models<jupyter_code>import os
import joblib
import numpy as np
import pandas as pd
from sklearn.pipeline import Pipeline
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.model_selection import learning_curve, GroupKFold

from optimalcodon.projects.rnastability.dataprocessing import get_data, general_preprocesing_pipeline
# 1 DATA PRE-PROCESSING

(train_x, train_y), (test_x, test_y) = get_data(
    '../data/191004-TrainAndTestSets/')
print("{} points for training and {} for testing with {} features".format(
    train_x.shape[0], test_x.shape[0], test_x.shape[1]))

groups = train_x.index.values<jupyter_output>67775 points for training and 7576 for testing with 6 features<jupyter_text>General cross-validation strategy<jupyter_code># function
def MYlearning_curve(mdl_id_name, train_x, train_y, estimator):
    """
    Args:
        mdl_id_name (str): id to identify model
        train_x: training predictors, should be pre-processed for the
            particular model to be evaluated
        train_y: training labels
        estimator: model
        cv: grouped k-fold
        
        
    """
    cv = GroupKFold(n_splits=5).split(train_x, train_y, groups=groups)
    
    print('generating learning curve for ' + mdl_id_name)
    
    train_sizes, train_scores, test_scores = learning_curve(
        estimator=estimator,
        cv=cv,
        X=train_x,
        y=train_y,
        train_sizes=np.linspace(0.1, 1, 5),
        scoring='r2',
        n_jobs=25,
        verbose=10
    )
    
    
    train_scores_mean = np.mean(train_scores, axis=1)
    train_scores_std = np.std(train_scores, axis=1)
    test_scores_mean = np.mean(test_scores, axis=1)
    test_scores_std = np.std(test_scores, axis=1)
    
    learning_curve_results = pd.DataFrame({
        'train_sizes': train_sizes,
        'train_scores_mean': train_scores_mean,
        'train_scores_std': train_scores_std,
        'test_scores_mean': test_scores_mean,
        'test_scores_std': test_scores_std
    })
    
    learning_curve_results['model'] = mdl_id_name
    return learning_curve_results<jupyter_output><empty_output><jupyter_text>*** Linear Models<jupyter_code>from sklearn.preprocessing import PolynomialFeatures, StandardScaler
from sklearn.linear_model import LinearRegression, Lasso, ElasticNet
from sklearn.feature_selection import VarianceThreshold
from sklearn.cross_decomposition import PLSRegression

linear_models = dict(
    lasso="../191004-TrainPredictiveModelsMrnaStability/results_data/trained_models/lasso.joblib",
    enet="../191004-TrainPredictiveModelsMrnaStability/results_data/trained_models/enet.joblib",
    linear="../191004-TrainPredictiveModelsMrnaStability/results_data/trained_models/linear_reg.joblib",
    pls ="../191004-TrainPredictiveModelsMrnaStability/results_data/trained_models/PLS.joblib"
)

# load the models
linear_models = {x:joblib.load(linear_models[x]) for x in linear_models}<jupyter_output><empty_output><jupyter_text>For this linear model I use a specific preprocessing pipeline to add 2nd degree polynomial. Next, I define this preprocessing pipeline.<jupyter_code># pre-processing

preprocessing = Pipeline([
    ('general', general_preprocesing_pipeline(train_x)), # see the code for general_preprocesing_pipeline
    ('polyfeaturs', PolynomialFeatures(degree=2)),
    ('zerovar', VarianceThreshold(threshold=0.0)),
    ('scaling', StandardScaler()) # I scale again not all polynomial features may be with scaled
])


preprocessing.fit(train_x)
train_x_transformed_for_linear = preprocessing.transform(train_x)
res_linear = []

for mdl_id, estimator in linear_models.items():
    tmp_res = MYlearning_curve(mdl_id, train_x_transformed_for_linear, train_y, estimator)
    res_linear.append(tmp_res)

res_linear = pd.concat(res_linear)
res_linear['type'] = "linear"
res_linear.to_csv('results-data/lc-linear.csv', index=False)<jupyter_output><empty_output><jupyter_text>*** Non-linear and Tree models<jupyter_code>non_linear_models = dict(
    knn="../191004-TrainPredictiveModelsMrnaStability/results_data/trained_models/knn.joblib",
    adaBoost="../191004-TrainPredictiveModelsMrnaStability/results_data/trained_models/AdaBoost.joblib",
    decisionTree="../191004-TrainPredictiveModelsMrnaStability/results_data/trained_models/decision tree.joblib",
    gbm="../191004-TrainPredictiveModelsMrnaStability/results_data/trained_models/gbm.joblib",
    randomforest="../191004-TrainPredictiveModelsMrnaStability/results_data/trained_models/random forest.joblib"
    
)
# load the models
non_linear_models = {x:joblib.load(non_linear_models[x]) for x in non_linear_models}

# reset the params for random forest
non_linear_models['randomforest'] = non_linear_models['randomforest'].set_params(n_jobs=1)<jupyter_output><empty_output><jupyter_text>For the next model we use the general pre-processing pipeline:<jupyter_code>general_pipeline = general_preprocesing_pipeline(train_x)
train_x_transformed = general_pipeline.transform(train_x)
res_nonlinear = []

for mdl_id, estimator in non_linear_models.items():
    tmp_res = MYlearning_curve(mdl_id, train_x_transformed, train_y, estimator)
    res_nonlinear.append(tmp_res)

res_nonlinear = pd.concat(res_nonlinear)
res_nonlinear['type'] = "non-linear"
res_nonlinear.to_csv('results-data/lc-nonlinear.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09729929707732149, 'fraction_numerical': 0.017203107658157604, 'mean_word_length': 5.3239766081871345, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '314324', 'license': 'MIT', 'n_tokens_mistral': 1880, 'n_tokens_neox': 1765, 'n_words': 316, 'path': 'paper-analysis/191005-EvaluateModelLearningCurve/01-learning_curve.ipynb', 'repo_name': 'santiago1234/MZT-rna-stability', 'size': '5406'}