<jupyter_start><jupyter_text>Tuning Tpot<jupyter_code>from tpot import TPOTClassifier
from sklearn.model_selection import train_test_split
import pandas as pd
import dask.dataframe as dd
from dask_cuda import LocalCUDACluster
from dask.distributed import Client
# CUDA Cluster Setup
cluster = LocalCUDACluster(enable_nvlink=True, rmm_pool_size="4GB")
client = Client(cluster)
client
# Load Data
train = pd.read_feather("/home/jovyan/work/ray_experiments/data/train_feat_1_baseline.feather")
target = train['target'] 
subtrain = train.drop(['ID_code', 'target'], axis = 1)
# convert to dask
#subtrain = dd.from_pandas(subtrain, npartitions=2)
X_train, X_test, y_train, y_test = train_test_split(subtrain, target, test_size=0.3,
                                                    stratify=target,
                                                    random_state=42)<jupyter_output><empty_output><jupyter_text>Initiating TPoTwe will use the cuml config<jupyter_code>import numpy as np

# This configuration provides users with access to a GPU the ability to
# use RAPIDS cuML and DMLC/XGBoost classifiers as estimators alongside
# the scikit-learn preprocessors in the TPOT default configuration.

classifier_config_cuml = {
    # cuML + DMLC/XGBoost Classifiers

    "cuml.neighbors.KNeighborsClassifier": {
        "n_neighbors": range(1, 101),
        "weights": ["uniform",],
    },

    "cuml.linear_model.LogisticRegression": {
        "penalty": ["l1", "l2", "elasticnet"],
        "C": [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.,],
    },
    
    #"cuml.ensemble.RandomForestClassifier": {
    #    "n_estimators": [100],
    #    "max_depth": np.arange(4,25,1),
    #    "max_features": range(3, 10),
    #    "n_bins": [8]
    #},

    "xgboost.XGBClassifier": {
        "n_estimators": [300],
        "max_depth": range(3, 10),
        "learning_rate": [1e-2, 1e-1, 0.5, 1.],
        "subsample": np.arange(0.05, 1.01, 0.05),
        "min_child_weight": range(1, 21),
        "alpha": [1, 10],
        "tree_method": ["gpu_hist"],
        "n_jobs": [1],
        "verbosity": [0]
    },
    
    # test lightgbm
    #"lightgbm.LGBMClassifier": {
    #    "n_estimators": [300],
    #    "learning_rate": [1e-2, 1e-1, 0.5, 1.],
    #    "objective": "binary",
    #    "min_child_weight": np.arange(1e-3, 1e-1, 1e-3),
    #    "n_jobs": [1]
    #},

    # Sklearn Preprocesssors

    "sklearn.preprocessing.Binarizer": {
        "threshold": np.arange(0.0, 1.01, 0.05)
    },

    # cuml pca instead
    #"cuml.PCA": {
    #    "n_components"
    #}
    
    # this is really slow
    #"sklearn.decomposition.FastICA": {
    #    "tol": np.arange(0.0, 1.01, 0.05)
    #},

    "sklearn.cluster.FeatureAgglomeration": {
        "linkage": ["ward", "complete", "average"],
        "affinity": ["euclidean", "l1", "l2", "manhattan", "cosine"]
    },

    "sklearn.preprocessing.MaxAbsScaler": {
    },

    "sklearn.preprocessing.MinMaxScaler": {
    },

    "sklearn.preprocessing.Normalizer": {
        "norm": ["l1", "l2", "max"]
    },

    "sklearn.kernel_approximation.Nystroem": {
        "kernel": ["rbf", "cosine", "chi2", "laplacian", "polynomial", "poly", "linear", "additive_chi2", "sigmoid"],
        "gamma": np.arange(0.0, 1.01, 0.05),
        "n_components": range(1, 11)
    },

    #"sklearn.decomposition.PCA": {
    #    "svd_solver": ["randomized"],
    #    "iterated_power": range(1, 11)
    #},
    
    "cuml.PCA": {
        "svd_solver": ["jacobi"],
        "iterated_power": range(1, 11)
    },

    "sklearn.kernel_approximation.RBFSampler": {
        "gamma": np.arange(0.0, 1.01, 0.05)
    },

    "sklearn.preprocessing.RobustScaler": {
    },

    "sklearn.preprocessing.StandardScaler": {
    },

    "tpot.builtins.ZeroCount": {
    },

    "tpot.builtins.OneHotEncoder": {
        "minimum_fraction": [0.05, 0.1, 0.15, 0.2, 0.25],
        "sparse": [False],
        "threshold": [10]
    },

    # Selectors

    "sklearn.feature_selection.SelectFwe": {
        "alpha": np.arange(0, 0.05, 0.001),
        "score_func": {
            "sklearn.feature_selection.f_classif": None
        }
    },

    "sklearn.feature_selection.SelectPercentile": {
        "percentile": range(1, 100),
        "score_func": {
            "sklearn.feature_selection.f_classif": None
        }
    },

    "sklearn.feature_selection.VarianceThreshold": {
        "threshold": [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.2]
    }
}
# Need njobs = 1 for cuml
pipeline_optimizer = TPOTClassifier(generations=5, population_size=10, cv=5, n_jobs=1,
                                    random_state=42, verbosity=2, scoring='roc_auc', 
                                    config_dict=classifier_config_cuml, 
                                    use_dask=True)<jupyter_output><empty_output><jupyter_text>Fittingneed to add dask `use_dask=True` in order to leverage multi GPU<jupyter_code>pipeline_optimizer.fit(X_train, y_train)<jupyter_output>[W] [11:17:57.996630] Warning(`_build_params`): As of v0.16, PCA invoked without an n_components argument defauts to using min(n_samples, n_features) rather than 1
{'fraction_non_alphanumeric': 0.15561174868702587, 'fraction_numerical': 0.04259871620307333, 'mean_word_length': 2.670235546038544, 'pattern_counts': {'":': 68, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '222759', 'license': 'Apache-2.0', 'n_tokens_mistral': 2057, 'n_tokens_neox': 1873, 'n_words': 339, 'path': 'Tuning_Tpot.ipynb', 'repo_name': 'Data-drone/ray_experiments', 'size': '5141'}