<jupyter_start><jupyter_text>Model Tuning  Standardized Scaling<jupyter_code>from sklearn.preprocessing import StandardScaler
ss = StandardScaler()

# list down features based on type
X_new = dataset.iloc[:, :3]
Y_new = dataset.iloc[:, 3]

# fit scaler on numeric features
ss.fit(X_new)

# scale numeric features now
X_scale = ss.transform(X_new)
X_new = pd.DataFrame(X_scale)

# Combine to new data frame

new_dataset = pd.concat([X_new, Y_new], axis=1)
# Split-out validation dataset
array = new_dataset.values
X_tune = array[:,0:3]
Y_tune = array[:,3]

validation_size = 0.20
seed = 7
X_train_new, X_validation_new, Y_train_new, Y_validation_new = model_selection.train_test_split(X_tune, Y_tune, test_size=validation_size, random_state=seed)

# Test options and evaluation metric
seed = 7
scoring = 'accuracy'
# Spot Check Algorithms
models = []
models.append(('LR', LogisticRegression()))
models.append(('LDA', LinearDiscriminantAnalysis()))
models.append(('KNN', KNeighborsClassifier()))
models.append(('CART', DecisionTreeClassifier()))
models.append(('NB', GaussianNB()))
models.append(('SVM', SVC()))
# evaluate each model in turn
results = []
names = []
for name, model in models:
	kfold = model_selection.KFold(n_splits=10, random_state=seed)
	cv_results = model_selection.cross_val_score(model, X_train_new, Y_train_new, cv=kfold, scoring=scoring)
	results.append(cv_results)
	names.append(name)
	msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std())
	print(msg)
# Make predictions on validation dataset
svmodel = SVC()
svmodel.fit(X_train_new, Y_train_new)
predictions = svmodel.predict(X_validation_new)
print(accuracy_score(Y_validation_new, predictions))
print(confusion_matrix(Y_validation_new, predictions))
print(classification_report(Y_validation_new, predictions))

y_true = np.array(Y_validation_new)[:,None]
y_probas = np.array(predictions)[:,None]
y_probas = np.concatenate((1-y_probas,y_probas),axis=1)
skplt.metrics.plot_roc(y_true, y_probas)
plt.show()<jupyter_output>0.8220524017467249
[[410  36]
 [127 343]]
             precision    recall  f1-score   support

        0.0       0.76      0.92      0.83       446
        1.0       0.91      0.73      0.81       470

avg / total       0.84      0.82      0.82       916<jupyter_text>MinMax Scaling<jupyter_code>from sklearn.preprocessing import MinMaxScaler
mms = MinMaxScaler()

# list down features based on type
X_new = dataset.iloc[:, :3]
Y_new = dataset.iloc[:, 3]

# fit scaler on numeric features
mms.fit(X_new)

# scale numeric features now
X_scale = mms.transform(X_new)
X_new = pd.DataFrame(X_scale)

# Combine to new data frame

new_dataset = pd.concat([X_new, Y_new], axis=1)
# Split-out validation dataset
array = new_dataset.values
X_tune = array[:,0:3]
Y_tune = array[:,3]

validation_size = 0.20
seed = 7
X_train_new, X_validation_new, Y_train_new, Y_validation_new = model_selection.train_test_split(X_tune, Y_tune, test_size=validation_size, random_state=seed)

# Test options and evaluation metric
seed = 7
scoring = 'accuracy'
# Spot Check Algorithms
models = []
models.append(('LR', LogisticRegression()))
models.append(('LDA', LinearDiscriminantAnalysis()))
models.append(('KNN', KNeighborsClassifier()))
models.append(('CART', DecisionTreeClassifier()))
models.append(('NB', GaussianNB()))
models.append(('SVM', SVC()))
# evaluate each model in turn
results = []
names = []
for name, model in models:
	kfold = model_selection.KFold(n_splits=10, random_state=seed)
	cv_results = model_selection.cross_val_score(model, X_train_new, Y_train_new, cv=kfold, scoring=scoring)
	results.append(cv_results)
	names.append(name)
	msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std())
	print(msg)<jupyter_output>LR: 0.572678 (0.065815)
LDA: 0.568306 (0.041902)
KNN: 0.813661 (0.012862)
CART: 0.796995 (0.018412)
NB: 0.804098 (0.014357)
SVM: 0.495082 (0.037459)<jupyter_text>Robust Scaling<jupyter_code>from sklearn.preprocessing import RobustScaler
rs = RobustScaler()

# list down features based on type
X_new = dataset.iloc[:, :3]
Y_new = dataset.iloc[:, 3]

# fit scaler on numeric features
rs.fit(X_new)

# scale numeric features now
X_scale = rs.transform(X_new)
X_new = pd.DataFrame(X_scale)

# Combine to new data frame

new_dataset = pd.concat([X_new, Y_new], axis=1)
# Split-out validation dataset
array = new_dataset.values
X_tune = array[:,0:3]
Y_tune = array[:,3]

validation_size = 0.20
seed = 7
X_train_new, X_validation_new, Y_train_new, Y_validation_new = model_selection.train_test_split(X_tune, Y_tune, test_size=validation_size, random_state=seed)

# Test options and evaluation metric
seed = 7
scoring = 'accuracy'
# Spot Check Algorithms
models = []
models.append(('LR', LogisticRegression()))
models.append(('LDA', LinearDiscriminantAnalysis()))
models.append(('KNN', KNeighborsClassifier()))
models.append(('CART', DecisionTreeClassifier()))
models.append(('NB', GaussianNB()))
models.append(('SVM', SVC()))
# evaluate each model in turn
results = []
names = []
for name, model in models:
	kfold = model_selection.KFold(n_splits=10, random_state=seed)
	cv_results = model_selection.cross_val_score(model, X_train_new, Y_train_new, cv=kfold, scoring=scoring)
	results.append(cv_results)
	names.append(name)
	msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std())
	print(msg)
# Make predictions on validation dataset
svmmodel = SVC()
svmmodel.fit(X_train_new, Y_train_new)
predictions = svmmodel.predict(X_validation_new)
print(accuracy_score(Y_validation_new, predictions))
print(confusion_matrix(Y_validation_new, predictions))
print(classification_report(Y_validation_new, predictions))

y_true = np.array(Y_validation_new)[:,None]
y_probas = np.array(predictions)[:,None]
y_probas = np.concatenate((1-y_probas,y_probas),axis=1)
skplt.metrics.plot_roc(y_true, y_probas)
plt.show()<jupyter_output>0.8220524017467249
[[410  36]
 [127 343]]
             precision    recall  f1-score   support

        0.0       0.76      0.92      0.83       446
        1.0       0.91      0.73      0.81       470

avg / total       0.84      0.82      0.82       916<jupyter_text>Hyperparameter Tuning<jupyter_code>from sklearn.model_selection import GridSearchCV
# setting the parameter grid
grid_parameters = {'kernel': ['linear', 'rbf', 'poly'],
 'gamma': [1e-3, 1e-4],
 'C': [100, 250, 500, 1000]}
# perform hyperparameter tuning
print("# Tuning hyper-parameters for accuracy\n")
clf = GridSearchCV(SVC(random_state=seed), grid_parameters, cv=5, scoring='accuracy')
clf.fit(X_train_new, Y_train_new)
# view accuracy scores for all the models
print("Grid scores for all the models based on CV:\n")
means = clf.cv_results_['mean_test_score']
stds = clf.cv_results_['std_test_score']
for mean, std, params in zip(means, stds, clf.cv_results_['params']):
 print("%0.5f (+/-%0.05f) for %r" % (mean, std * 2, params))
# check out best model performance
print("\nBest parameters set found on development set:", clf.best_params_)
print("Best model validation accuracy:", clf.best_score_)
import scipy
from sklearn.model_selection import RandomizedSearchCV
param_grid = {'C': scipy.stats.expon(scale=10),
 'gamma': scipy.stats.expon(scale=.1),
 'kernel': ['rbf', 'linear']}
random_search = RandomizedSearchCV(SVC(random_state=seed), param_distributions=param_grid,
 n_iter=50, cv=5)
random_search.fit(X_train_new, Y_train_new)
print("Best parameters set found on development set:")
random_search.best_params_
# get best model, predict and evaluate performance
rs_best = random_search.best_estimator_
rs_predictions = rs_best.predict(X_validation_new)
print(accuracy_score(Y_validation_new, rs_predictions))
print(confusion_matrix(Y_validation_new, rs_predictions))
print(classification_report(Y_validation_new, rs_predictions))<jupyter_output>0.8307860262008734
[[405  41]
 [114 356]]
             precision    recall  f1-score   support

        0.0       0.78      0.91      0.84       446
        1.0       0.90      0.76      0.82       470

avg / total       0.84      0.83      0.83       916
{'fraction_non_alphanumeric': 0.12778677294806326, 'fraction_numerical': 0.045335658238884045, 'mean_word_length': 5.21998450813323, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '152841', 'license': 'Apache-2.0', 'n_tokens_mistral': 3092, 'n_tokens_neox': 2882, 'n_words': 634, 'path': 'data-challenge-efishery/Model Mk I.ipynb', 'repo_name': 'lppratama/data-science', 'size': '8029'}