<jupyter_start><jupyter_text>**Section:** Model Selection<jupyter_code>import os
import itertools as it
import warnings
import time
import pickle

import numpy as np

import pandas as pd

%matplotlib inline
from matplotlib import pyplot as plt
from matplotlib import cm
import seaborn as sns

import joblib

import pathlib

from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.exceptions import DataConversionWarning

from sklearn import metrics

from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report

import tqdm

from libs.container import Container
from libs.nearest import nearest
from libs.experiment import WithAnotherExperiment, roc, metrics
from libs.precstar import  prec_star

warnings.simplefilter("ignore", category=DataConversionWarning)
PATH = pathlib.Path(os.path.abspath(os.path.dirname("")))

DATA_PATH = PATH / "_data" / "s20k_scaled.pkl.bz2"
DATA_PATH = PATH / "bkp" / "s20k_scaled.pkl.bz2"

COLUMNS_NO_FEATURES = ['id', 'tile', 'cnt', 'ra_k', 'dec_k', 'vs_type', 'vs_catalog', 'cls'] 
sample = pd.read_pickle(DATA_PATH)

# the features
X_columns = [c for c in sample.columns if c not in COLUMNS_NO_FEATURES]
y_column = "cls"

sample[X_columns] =  sample[X_columns].astype(np.float32)

data = Container({k: v for k, v in sample.groupby("tile") if k in ["b234", "b360", "b278", "b261"]})

del sample
len(X_columns)
def score_func(y, y_prob, **kwargs):
    prec, rec, thr = metrics.precision_recall_curve(
            y, y_prob, sample_weight=Nonbkp/
    idx = nearest(array=rec, value=.9)
    return prec[idx]


def grid_search(data, estimator, param_grid):
    print(f"Running {type(estimator)}")
    clf = GridSearchCV(
        estimator, 
        param_grid, 
        cv=5, scoring=metrics.make_scorer(score_func, needs_proba=True), n_jobs=-1)

    X, y = data[X_columns].values, data.cls.values

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

    clf.fit(X_train, y_train)

    print("Best parameters set found on development set:")
    print()
    print(clf.best_params_)

    print()
    print("Grid scores on development set:")
    print()
    means = clf.cv_results_['mean_test_score']
    stds = clf.cv_results_['std_test_score']
    for mean, std, params in zip(means, stds, clf.cv_results_['params']):
        print("%0.3f (+/-%0.03f) for %r"
              % (mean, std * 2, params))
    print()

    print("Detailed classification report:")
    print()
    print("The model is trained on the full development set.")
    print("The scores are computed on the full evaluation set.")
    print()
    y_true, y_pred = y_test, clf.predict(X_test)
    print(classification_report(y_true, y_pred))
    print()

    return clf<jupyter_output><empty_output><jupyter_text>SVM-RBF<jupyter_code>gamma_range = np.array([1.e-4, 3.e-4, 1.e+3, 3.e-3])
%%time
svc_rbf = grid_search(
    data=data.b278, 
    estimator=SVC(),
    param_grid=[{
        'kernel': ['rbf'], 
        'C': [1, 10, 30, 50, 100], 
        "gamma": gamma_range, 
        "probability": [True]}])<jupyter_output>Running <class 'sklearn.svm._classes.SVC'>
Best parameters set found on development set:

{'C': 10, 'gamma': 0.003, 'kernel': 'rbf', 'probability': True}

Grid scores on development set:

0.245 (+/-0.143) for {'C': 1, 'gamma': 0.0001, 'kernel': 'rbf', 'probability': True}
0.251 (+/-0.151) for {'C': 1, 'gamma': 0.0003, 'kernel': 'rbf', 'probability': True}
0.022 (+/-0.000) for {'C': 1, 'gamma': 1000.0, 'kernel': 'rbf', 'probability': True}
0.343 (+/-0.161) for {'C': 1, 'gamma': 0.003, 'kernel': 'rbf', 'probability': True}
0.258 (+/-0.161) for {'C': 10, 'gamma': 0.0001, 'kernel': 'rbf', 'probability': True}
0.298 (+/-0.220) for {'C': 10, 'gamma': 0.0003, 'kernel': 'rbf', 'probability': True}
0.022 (+/-0.000) for {'C': 10, 'gamma': 1000.0, 'kernel': 'rbf', 'probability': True}
0.388 (+/-0.144) for {'C': 10, 'gamma': 0.003, 'kernel': 'rbf', 'probability': True}
0.283 (+/-0.218) for {'C': 30, 'gamma': 0.0001, 'kernel': 'rbf', 'probability': True}
0.306 (+/-0.177) for {'C': 30, 'gamma': 0.00[...]<jupyter_text>SVM-Linear<jupyter_code>%%time

Cs = [20, 30, 40,  50, 60, 100]

svc_linear = grid_search(
    data=data.b278, 
    estimator=SVC(probability=True),
    param_grid=[{'kernel': ['linear'], 'C': Cs, "probability": [True]}])<jupyter_output>Running <class 'sklearn.svm._classes.SVC'>
Best parameters set found on development set:

{'C': 50, 'kernel': 'linear', 'probability': True}

Grid scores on development set:

0.247 (+/-0.199) for {'C': 20, 'kernel': 'linear', 'probability': True}
0.243 (+/-0.205) for {'C': 30, 'kernel': 'linear', 'probability': True}
0.247 (+/-0.198) for {'C': 40, 'kernel': 'linear', 'probability': True}
0.251 (+/-0.203) for {'C': 50, 'kernel': 'linear', 'probability': True}
0.251 (+/-0.203) for {'C': 60, 'kernel': 'linear', 'probability': True}
0.246 (+/-0.195) for {'C': 100, 'kernel': 'linear', 'probability': True}

Detailed classification report:

The model is trained on the full development set.
The scores are computed on the full evaluation set.

              precision    recall  f1-score   support

           0       1.00      1.00      1.00      4005
           1       0.90      0.77      0.83        83

    accuracy                           0.99      4088
   macro avg       0.95      0.88    [...]<jupyter_text>KNN<jupyter_code>k_range = [4, 5, 6]
k_range
%%time
knn = grid_search(
    data=data.b278, 
    estimator=KNeighborsClassifier(),
    param_grid=[{
        "weights": ['uniform', 'distance'], 
        "algorithm": ['auto'],
        "p": [1, 2, 3],
        "n_neighbors": k_range}])<jupyter_output>Running <class 'sklearn.neighbors._classification.KNeighborsClassifier'>
Best parameters set found on development set:

{'algorithm': 'auto', 'n_neighbors': 5, 'p': 1, 'weights': 'uniform'}

Grid scores on development set:

0.493 (+/-0.473) for {'algorithm': 'auto', 'n_neighbors': 3, 'p': 1, 'weights': 'uniform'}
0.493 (+/-0.473) for {'algorithm': 'auto', 'n_neighbors': 3, 'p': 1, 'weights': 'distance'}
0.335 (+/-0.515) for {'algorithm': 'auto', 'n_neighbors': 3, 'p': 2, 'weights': 'uniform'}
0.335 (+/-0.515) for {'algorithm': 'auto', 'n_neighbors': 3, 'p': 2, 'weights': 'distance'}
0.327 (+/-0.499) for {'algorithm': 'auto', 'n_neighbors': 3, 'p': 3, 'weights': 'uniform'}
0.327 (+/-0.499) for {'algorithm': 'auto', 'n_neighbors': 3, 'p': 3, 'weights': 'distance'}
0.510 (+/-0.070) for {'algorithm': 'auto', 'n_neighbors': 5, 'p': 1, 'weights': 'uniform'}
0.510 (+/-0.070) for {'algorithm': 'auto', 'n_neighbors': 5, 'p': 1, 'weights': 'distance'}
0.438 (+/-0.042) for {'algorithm': 'auto', '[...]<jupyter_text>Random Forest<jupyter_code>%%time

rf = grid_search(
    data=data.b278, 
    estimator=RandomForestClassifier(),
    param_grid=[{
        'max_features': ['auto', 'sqrt', "log2", None, 0.2, 0.5], 
        "min_samples_split": [2, 5, 10],
        "n_estimators": [500], 
        "criterion": ["entropy"], 
        "n_jobs": [-1]}])

del rf.best_params_["n_jobs"]
rf.best_params_
best_params = {
    "rf": rf.best_params_,
    "knn": knn.best_params_,
    "svml": svc_linear.best_params_,
    "svmr": svc_rbf.best_params_
}

joblib.dump(best_params, "_cache/best_params.pkl.bz2", compress=3)
best_params
import datetime
datetime.datetime.now()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17986666666666667, 'fraction_numerical': 0.0584, 'mean_word_length': 3.944627554383652, 'pattern_counts': {'":': 15, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '646358', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3099, 'n_tokens_neox': 2760, 'n_words': 728, 'path': '01_section_model_best_params.ipynb', 'repo_name': 'leliel12/How-far-can-we-can-go', 'size': '7500'}