<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import numpy as np
import io
import gc
import time
from pprint import pprint
from datetime import date

# settings
import warnings
warnings.filterwarnings("ignore")
gc.enable()
# !pip3 install xgboost > /dev/null
!pip3 install tune-sklearn ray[tune] > /dev/null
# Global Variables
random_state = 50
# connect to google drive
from google.colab import drive
drive.mount('/content/drive')
gDrivePath = '/content/drive/MyDrive/Datasets/Hackerearth_genetic_testing/dataset/'
df_train = pd.read_csv(gDrivePath+'train_preprocessed.csv')
df_test = pd.read_csv(gDrivePath+'test_preprocessed.csv')
df_train.shape
df_test.shape
df_train.sample(3)<jupyter_output><empty_output><jupyter_text>Checking if the dataset is balanced/imbalanced - Genetic Disorder<jupyter_code>target_count = df_train['Genetic Disorder'].value_counts()
target_count<jupyter_output><empty_output><jupyter_text>Checking if the dataset is balanced/imbalanced - Disorder Subclass<jupyter_code>target_count = df_train['Disorder Subclass'].value_counts()
target_count<jupyter_output><empty_output><jupyter_text>Splitting Data into train-cv<jupyter_code>genetic_disorder_labels = df_train['Genetic Disorder'].values
disorder_subclass_labels = df_train['Disorder Subclass'].values

df_train.drop(['Genetic Disorder','Disorder Subclass'], axis=1, inplace=True)
df_test.drop(['Genetic Disorder','Disorder Subclass'], axis=1, inplace=True, errors='ignore')
# classification split for genetic_disorder_labels
from sklearn.model_selection import train_test_split
X_train_genetic_disorder, X_cv_genetic_disorder, y_train_genetic_disorder, y_cv_genetic_disorder = train_test_split(df_train, genetic_disorder_labels, test_size=0.1, random_state=random_state)

# classification split for disorder_subclass_labels
X_train_disorder_subclass, X_cv_disorder_subclass, y_train_disorder_subclass, y_cv_disorder_subclass = train_test_split(df_train, disorder_subclass_labels, test_size=0.1, random_state=random_state)<jupyter_output><empty_output><jupyter_text>Over Sampling using SMOTE for Genetic Disorder<jupyter_code># https://machinelearningmastery.com/smote-oversampling-for-imbalanced-classification/
from imblearn.over_sampling import SMOTE
smote_overSampling = SMOTE()
X_train_genetic_disorder,y_train_genetic_disorder = smote_overSampling.fit_resample(X_train_genetic_disorder,y_train_genetic_disorder)
unique, counts = np.unique(y_train_genetic_disorder, return_counts=True)
dict(zip(unique, counts))<jupyter_output><empty_output><jupyter_text>Over Sampling using SMOTE for Disorder Subclass<jupyter_code># https://machinelearningmastery.com/smote-oversampling-for-imbalanced-classification/
from imblearn.over_sampling import SMOTE
smote_overSampling = SMOTE()
X_train_disorder_subclass,y_train_disorder_subclass = smote_overSampling.fit_resample(X_train_disorder_subclass,y_train_disorder_subclass)
unique, counts = np.unique(y_train_disorder_subclass, return_counts=True)
dict(zip(unique, counts))<jupyter_output><empty_output><jupyter_text>Scaling data : genetic_disorder<jupyter_code>from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
X_train_genetic_disorder_scaled = scaler.fit_transform(X_train_genetic_disorder)
X_cv_genetic_disorder_scaled = scaler.transform(X_cv_genetic_disorder)

X_test_scaled = scaler.transform(df_test)

# X_train_genetic_disorder_scaled<jupyter_output><empty_output><jupyter_text>Scaling data : disorder_subclass<jupyter_code>from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
X_train_disorder_subclass_scaled = scaler.fit_transform(X_train_disorder_subclass)
X_cv_disorder_subclass_scaled = scaler.transform(X_cv_disorder_subclass)

X_test_scaled = scaler.transform(df_test)

# X_train_disorder_subclass_scaled<jupyter_output><empty_output><jupyter_text>Modelling & Cross-Validation for genetic_disorder<jupyter_code># %%time
# # Train multiple models : https://www.kaggle.com/tflare/testing-multiple-models-with-scikit-learn-0-79425
# from sklearn.linear_model import LogisticRegression
# from sklearn.svm import SVC, LinearSVC
# from sklearn.neighbors import KNeighborsClassifier
# from sklearn.tree import DecisionTreeClassifier
# from sklearn.ensemble import RandomForestClassifier
# from sklearn.ensemble import AdaBoostClassifier
# from sklearn.ensemble import BaggingClassifier
# from sklearn.ensemble import ExtraTreesClassifier
# from sklearn.ensemble import GradientBoostingClassifier
# from sklearn.linear_model import LogisticRegressionCV
# from xgboost import XGBClassifier

# from sklearn.model_selection import cross_val_score

# models = []

# # LogisticRegression = LogisticRegression(n_jobs=-1)
# # LinearSVC = LinearSVC()
# # KNeighbors = KNeighborsClassifier(n_jobs=-1)
# # DecisionTree = DecisionTreeClassifier()
# # AdaBoost = AdaBoostClassifier()
# # Bagging = BaggingClassifier()
# # GradientBoosting = GradientBoostingClassifier()
# # LogisticRegressionCV = LogisticRegressionCV(n_jobs=-1)
# # XGBClassifier = XGBClassifier(nthread=-1)
# RandomForest = RandomForestClassifier()
# ExtraTrees = ExtraTreesClassifier()

# # models.append(("LogisticRegression",LogisticRegression))
# # models.append(("LinearSVC", LinearSVC))
# # models.append(("KNeighbors", KNeighbors))
# # models.append(("DecisionTree", DecisionTree))
# # models.append(("AdaBoost", AdaBoost))
# # models.append(("Bagging", Bagging))
# # models.append(("GradientBoosting", GradientBoosting))
# # models.append(("LogisticRegressionCV", LogisticRegressionCV))
# # models.append(("XGBClassifier", XGBClassifier))
# models.append(("RandomForest", RandomForest))
# models.append(("ExtraTrees", ExtraTrees))

# # metric_names = ['f1', 'average_precision', 'accuracy', 'precision', 'recall']
# metric_names = ['f1_weighted']
# results = []
# names = []

# nested_dict = {}

# for name,model in models:
#   nested_dict[name] = {}
#   for metric in metric_names:
#     print("\nRunning : {}, with metric : {}".format(name, metric))
#     score = cross_val_score(model, X_train_genetic_disorder_scaled, y_train_genetic_disorder, n_jobs=-1, scoring=metric, cv=5)
#     nested_dict[name][metric] = score.mean()
# import json
# print(json.dumps(nested_dict, sort_keys=True, indent=4))<jupyter_output><empty_output><jupyter_text>Modelling & Cross-Validation for disorder_subclass<jupyter_code># %%time
# # Train multiple models : https://www.kaggle.com/tflare/testing-multiple-models-with-scikit-learn-0-79425
# from sklearn.linear_model import LogisticRegression
# from sklearn.svm import SVC, LinearSVC
# from sklearn.neighbors import KNeighborsClassifier
# from sklearn.tree import DecisionTreeClassifier
# from sklearn.ensemble import RandomForestClassifier
# from sklearn.ensemble import AdaBoostClassifier
# from sklearn.ensemble import BaggingClassifier
# from sklearn.ensemble import ExtraTreesClassifier
# from sklearn.ensemble import GradientBoostingClassifier
# from sklearn.linear_model import LogisticRegressionCV
# from xgboost import XGBClassifier

# from sklearn.model_selection import cross_val_score

# models = []

# # LogisticRegression = LogisticRegression(n_jobs=-1)
# # LinearSVC = LinearSVC()
# # KNeighbors = KNeighborsClassifier(n_jobs=-1)
# # DecisionTree = DecisionTreeClassifier()
# # AdaBoost = AdaBoostClassifier()
# # Bagging = BaggingClassifier()
# # GradientBoosting = GradientBoostingClassifier()
# # LogisticRegressionCV = LogisticRegressionCV(n_jobs=-1)
# # XGBClassifier = XGBClassifier(nthread=-1)
# RandomForest = RandomForestClassifier()
# ExtraTrees = ExtraTreesClassifier()

# # models.append(("LogisticRegression",LogisticRegression))
# # models.append(("LinearSVC", LinearSVC))
# # models.append(("KNeighbors", KNeighbors))
# # models.append(("DecisionTree", DecisionTree))
# # models.append(("AdaBoost", AdaBoost))
# # models.append(("Bagging", Bagging))
# # models.append(("GradientBoosting", GradientBoosting))
# # models.append(("LogisticRegressionCV", LogisticRegressionCV))
# # models.append(("XGBClassifier", XGBClassifier))
# models.append(("RandomForest", RandomForest))
# models.append(("ExtraTrees", ExtraTrees))

# # metric_names = ['f1', 'average_precision', 'accuracy', 'precision', 'recall']
# metric_names = ['f1_weighted']
# results = []
# names = []

# nested_dict = {}

# for name,model in models:
#   nested_dict[name] = {}
#   for metric in metric_names:
#     print("\nRunning : {}, with metric : {}".format(name, metric))
#     score = cross_val_score(model, X_train_disorder_subclass_scaled, y_train_disorder_subclass, n_jobs=-1, scoring=metric, cv=5)
#     nested_dict[name][metric] = score.mean()
# import json
# print(json.dumps(nested_dict, sort_keys=True, indent=4))<jupyter_output><empty_output><jupyter_text>Hyperparameter tuning  Tuning for : genetic_disorder<jupyter_code># from sklearn.model_selection import GridSearchCV
from tune_sklearn import TuneGridSearchCV

from sklearn.ensemble import ExtraTreesClassifier

# model_classifier = ExtraTreesClassifier(max_depth=15, n_estimators=400)
model_classifier = ExtraTreesClassifier(criterion='gini', bootstrap=False, max_features='auto', warm_start=False)


# # Best Params: {'criterion': 'gini', 'max_depth': 15, 'bootstrap': False, 'max_features': 'auto', 'warm_start': False, 'n_estimators': 400}
# Parameters to tune:
parameters = {
    'n_estimators': np.arange(100, 3000, 100, dtype=int),
    'max_depth': np.arange(5, 16, 1, dtype=int),
    # 'criterion': ['gini', 'entropy'],
    # 'bootstrap': [True, False],
    # 'max_features': ['auto', 'sqrt', 'log2'],
    # 'warm_start': [True, False],
}
tune_search_genetic_disorder = TuneGridSearchCV(
    model_classifier,
    parameters,
    scoring='f1_weighted',
    verbose=1,
    n_jobs=-1,
)
tune_search_genetic_disorder.fit(X_train_genetic_disorder_scaled, y_train_genetic_disorder)

pred = tune_search_genetic_disorder.predict(X_cv_genetic_disorder_scaled)
accuracy = np.count_nonzero(np.array(pred) == np.array(y_cv_genetic_disorder)) / len(pred)
print("Tune Accuracy:", accuracy)
print("Best Params:", tune_search_genetic_disorder.best_params_)<jupyter_output><empty_output><jupyter_text>Tuning for : disorder_subclass<jupyter_code># from sklearn.model_selection import GridSearchCV
from tune_sklearn import TuneGridSearchCV

from sklearn.ensemble import ExtraTreesClassifier

# model_classifier = ExtraTreesClassifier(max_depth=15, n_estimators=400)
model_classifier = ExtraTreesClassifier(criterion='entropy', bootstrap=False, max_features='log2', warm_start=True)

## Best Params: {'criterion': 'entropy', 'max_depth': 15, 'bootstrap': False, 'max_features': 'log2', 'warm_start': True}
# Parameters to tune:
parameters = {
    'n_estimators': np.arange(100, 3000, 100, dtype=int),
    'max_depth': np.arange(5, 16, 1, dtype=int),
    # 'criterion': ['gini', 'entropy'],
    # 'bootstrap': [True, False],
    # 'max_features': ['auto', 'sqrt', 'log2'],
    # 'warm_start': [True, False],
}
tune_search_disorder_subclass = TuneGridSearchCV(
    model_classifier,
    parameters,
    scoring='f1_weighted',
    verbose=1,
    n_jobs=-1,
)
tune_search_disorder_subclass.fit(X_train_disorder_subclass_scaled, y_train_disorder_subclass)
# Check accuracy
pred = tune_search_disorder_subclass.predict(X_cv_disorder_subclass_scaled)
accuracy = np.count_nonzero(np.array(pred) == np.array(y_cv_disorder_subclass)) / len(pred)
print("Tune Accuracy:", accuracy)
print("Best Params:", tune_search_disorder_subclass.best_params_)
import joblib
joblib.dump(tune_search_genetic_disorder, 'genetic_disorder_model.pkl')
joblib.dump(tune_search_disorder_subclass, 'disorder_subclass_model.pkl')
trained_model_genetic_disorder = joblib.load('genetic_disorder_model.pkl')
trained_model_disorder_subclass = joblib.load('disorder_subclass_model.pkl')<jupyter_output><empty_output><jupyter_text>Predicting on CV data<jupyter_code>#<jupyter_output><empty_output><jupyter_text>Predicting on test Data<jupyter_code>predictions_genetic_disorder_test = trained_model_genetic_disorder.predict(X_test_scaled)
predictions_disorder_subclass_test = trained_model_disorder_subclass.predict(X_test_scaled)
len(predictions_genetic_disorder_test)
len(predictions_disorder_subclass_test)
read = pd.read_csv(gDrivePath + 'test.csv')
read.shape
submission = pd.DataFrame({
        "Patient Id": read["Patient Id"],
        "Genetic Disorder": predictions_genetic_disorder_test,
        "Disorder Subclass": predictions_disorder_subclass_test,
    })
submission.head()
submission.to_csv('submission.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11447891805887032, 'fraction_numerical': 0.0076372315035799524, 'mean_word_length': 8.318754633061527, 'pattern_counts': {'":': 3, '<': 57, '<?xml version=': 0, '>': 59, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '94207', 'license': 'MIT', 'n_tokens_mistral': 4463, 'n_tokens_neox': 4306, 'n_words': 685, 'path': 'Hackerearth-Predict-the-genetic-disorders/2_genetic_testing_modelling.ipynb', 'repo_name': 'chiranjeet14/ML_Projects', 'size': '12570'}