<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import warnings
warnings.filterwarnings('ignore')

%matplotlib inline
plt.style.use('seaborn-whitegrid')<jupyter_output><empty_output><jupyter_text>Load Dataframe<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
# /content/drive/My Drive/Colab Notebooks/training_sampled_data.csv
raw_df = pd.read_csv('/content/drive/My Drive/Colab Notebooks/training_sampled_data.csv')
print('Shape of dataframe: ', raw_df.shape)
raw_df.head()
# dropping rows with ' --'
raw_df_1 = raw_df[raw_df.label != ' --']
raw_df_1_copy = raw_df_1.copy()
print('Shape of dataframe: ', raw_df_1.shape)
raw_df_1.info()
raw_df_1.describe().T
raw_df_1['label'].value_counts().plot.bar()
from sklearn.preprocessing import LabelEncoder

labelencoder = LabelEncoder()
raw_df_1['label'] = labelencoder.fit_transform(raw_df_1['label'])
raw_df_1.head()<jupyter_output><empty_output><jupyter_text>Correlation matrix and Heatmaps<jupyter_code>corr = raw_df_1.corr()

fig = plt.figure(figsize = (25,10))
ax = fig.add_subplot(1,1,1)
cax = ax.matshow(corr,cmap='coolwarm', vmin=-1, vmax=1)
fig.colorbar(cax)
ticks = np.arange(0,len(raw_df_1.columns),1)
ax.set_xticks(ticks)
plt.xticks(rotation=90)
ax.set_yticks(ticks)
ax.set_xticklabels(raw_df_1.columns)
ax.set_yticklabels(raw_df_1.columns)
plt.show()
plt.figure(figsize=(20,20))
sns.heatmap(raw_df_1.corr(), annot=True)
plt.show()<jupyter_output><empty_output><jupyter_text>Handling Imbalance<jupyter_code>sns.countplot(raw_df_1['label'], label = 'Count')
plt.show()
# ['above average': '0',
#  'below average': '2',
#  'average': '1']


# class count
class_count_0, class_count_1, class_count_2 = raw_df_1['label'].value_counts()

# Separate class
class_0 = raw_df_1[raw_df_1['label'] == 0]
class_1 = raw_df_1[raw_df_1['label'] == 1]
class_2 = raw_df_1[raw_df_1['label'] == 2]

# print the shape of the class
print('above average:', class_0.shape)
print('average:', class_1.shape)
print('below average:', class_2.shape)
import imblearn
from imblearn.under_sampling import TomekLinks
from imblearn.over_sampling import SMOTE
from imblearn.under_sampling import NearMiss
from collections import Counter 
raw_df_1_array = raw_df_1.values 
X = raw_df_1.iloc[:,:-1]
y = raw_df_1.iloc[:,-1:]
tome_links = TomekLinks(sampling_strategy = 'majority')

x_tome_links, y_tome_links = tome_links.fit_resample(X, y)

print('Original dataset shape', (y.value_counts()))
print('Resample dataset shape', Counter(y_tome_links))
smote = SMOTE()

x_smote, y_smote = smote.fit_resample(X, y)
print('Original dataset shape', (y.value_counts()))
print('Resample dataset shape', Counter(y_smote))
near_miss = NearMiss()
x_near_miss, y_near_miss = near_miss.fit_resample(X, y)
print('Original dataset shape', (y.value_counts()))
print('Resample dataset shape', Counter(y_near_miss))<jupyter_output>Original dataset shape label
1        414
0        354
2        101
dtype: int64
Resample dataset shape Counter({0: 101, 1: 101, 2: 101})<jupyter_text>Machine Learning Algorithms<jupyter_code>from xgboost import XGBClassifier
from sklearn.metrics import scorer, accuracy_score, f1_score, confusion_matrix, recall_score, precision_score, roc_auc_score
from sklearn.model_selection import train_test_split
def model_result(X_train, X_test, y_train, y_test, model):
  model_1 = model
  model_1.fit(X_train, y_train)
  precise_score = precision_score(y_true = y_test, y_pred = model_1.predict(X_test),average = 'macro')
  rec_score = recall_score(y_true = y_test, y_pred = model_1.predict(X_test),average = 'macro',)
  print('Precision score: {}% '.format(round(precise_score*100, 2)))
  print('Recall score: {}% '.format(round(rec_score*100, 2)))
  sns.heatmap(confusion_matrix(y_true = y_test, y_pred = model_1.predict(X_test)), annot = True)
  plt.show()


X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
X_train_near_miss, X_test_near_miss, y_train_near_miss, y_test_near_miss = train_test_split(x_near_miss, y_near_miss, test_size=0.2, random_state=42)
X_train_tome_links, X_test_tome_links, y_train_tome_links, y_test_tome_links = train_test_split(x_tome_links, y_tome_links, test_size=0.2, random_state=42)
X_train_smote, X_test_smote, y_train_smote, y_test_smote = train_test_split(x_smote, y_smote, test_size=0.2, random_state=42)<jupyter_output><empty_output><jupyter_text>* __Recall_score__: Is the proportion of correct predicted labels to the total number of actual labels, averaged over instances.* __Precision_score__:  The precision is intuitively the ability of the classifier not to label as positive a sample that is negative.<jupyter_code>print('Results of imbbalanced dataset: ')
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
model_result(X_train, X_test, y_train, y_test, model = XGBClassifier())
print()
print('Results of nearMiss balancing: ')
X_train, X_test, y_train, y_test = train_test_split(x_near_miss, y_near_miss, test_size=0.2, random_state=42)
model_result(X_train, X_test, y_train, y_test, model = XGBClassifier())
print()
print('Results of TomeLinks balancing: ')
X_train, X_test, y_train, y_test = train_test_split(x_tome_links, y_tome_links, test_size=0.2, random_state=42)
model_result(X_train, X_test, y_train, y_test, model = XGBClassifier())
print()
print('Results of SMOTE balancing: ')
X_train, X_test, y_train, y_test = train_test_split(x_smote, y_smote, test_size=0.2, random_state=42)
model_result(X_train, X_test, y_train, y_test, model = XGBClassifier())<jupyter_output>Results of imbbalanced dataset: 
Precision score: 84.39% 
Recall score: 69.47%<jupyter_text>* Recall score is highest for imbalance dataset handled by SMOTE for XGBClassifier model.* Recall score is more reliable, the precision score is highest for imbalanced dataset, this may be due to the fact it is able to predict above average correctly, since this is a significant number, it may not give the true picture.  Artificial Neural Network<jupyter_code>import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import KFold
X = raw_df_1_copy.iloc[:,:-1]
y = raw_df_1_copy.iloc[:,-1:]
# encode class values as integers
encoder = LabelEncoder()
encoder.fit(y)
encoded_Y = encoder.transform(y)
# convert integers to dummy variables (i.e. one hot encoded)
dummy_y = tf.keras.utils.to_categorical(encoded_Y)
 
# function for sequential model
# input dimension is the number of features
# number of categories in output is 3
def sequential_model():
  model = Sequential()
  model.add(Dense(64, input_dim = 12, activation = 'relu'))
  model.add(Dropout(rate = 0.2))
  model.add(Dense(3, activation = 'softmax'))
  # compiling the model
  model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
  return model
 
estimator = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn = sequential_model, 
                                                           epochs = 100, 
                                                           batch_size = 32, 
                                                           verbose = 0)
kfold = KFold(n_splits=10, 
              shuffle=True)
results = cross_val_score(estimator, 
                          X, 
                          dummy_y, 
                          cv=kfold,)
print("Average accuracy: %.2f%%, Standard deviation across results: %.2f%%" % (results.mean()*100, results.std()*100))
# x_smote, y_smote
X_train, X_test, y_train, y_test = train_test_split(x_smote, y_smote, test_size=0.1, random_state=42)
dummy_y_train = tf.keras.utils.to_categorical(y_train)
dummy_y_test = tf.keras.utils.to_categorical(y_test)
ann_model = sequential_model()
ann_model.fit(X_train, dummy_y_train, epochs = 500, batch_size=32, validation_data= (X_test, dummy_y_test),)<jupyter_output>Epoch 1/500
35/35 [==============================] - 0s 5ms/step - loss: 1.0996 - accuracy: 0.3402 - val_loss: 1.0705 - val_accuracy: 0.4560
Epoch 2/500
35/35 [==============================] - 0s 2ms/step - loss: 1.0486 - accuracy: 0.4736 - val_loss: 1.0503 - val_accuracy: 0.4160
Epoch 3/500
35/35 [==============================] - 0s 3ms/step - loss: 1.0178 - accuracy: 0.4933 - val_loss: 1.0355 - val_accuracy: 0.3920
Epoch 4/500
35/35 [==============================] - 0s 2ms/step - loss: 0.9882 - accuracy: 0.5004 - val_loss: 1.0244 - val_accuracy: 0.3920
Epoch 5/500
35/35 [==============================] - 0s 3ms/step - loss: 0.9670 - accuracy: 0.5300 - val_loss: 1.0164 - val_accuracy: 0.4160
Epoch 6/500
35/35 [==============================] - 0s 3ms/step - loss: 0.9545 - accuracy: 0.5237 - val_loss: 1.0072 - val_accuracy: 0.4320
Epoch 7/500
35/35 [==============================] - 0s 3ms/step - loss: 0.9362 - accuracy: 0.5542 - val_loss: 1.0003 - val_accuracy: 0.4560
Epoch 8/500
3[...]<jupyter_text>Prediction<jupyter_code>def predict_risk(h1_score, h2_score, h3_score, h4_score, h5_score, h6_score, text_score, title_score, desc_score, domain_score, path_score, alt_img_score, model):
    df = pd.DataFrame.from_dict({'h1_score': [h1_score], 
                                 'h2_score': [h2_score], 
                                 'h3_score': [h3_score],
                                 'h4_score': [h4_score], 
                                 'h5_score': [h5_score], 
                                 'h6_score': [h6_score],
                                 'text_score': [text_score], 
                                 'title_score': [title_score], 
                                 'desc_score': [desc_score],
                                 'domain_score': [domain_score], 
                                 'path_score': [path_score], 
                                 'alt_img_score': [alt_img_score]})
    
    array_file = df.to_numpy()
    pred = model.predict_proba(array_file)[0]
    print(f'The probability of above average is {round(pred[0]*100,2)}%')
    print(f'probability of below average is {round(pred[2]*100,2)}%')
    print(f'probability of average is {round(pred[1]*100,2)}%') 
                                                                                                                               
X_train, X_test, y_train, y_test = train_test_split(x_smote, y_smote, test_size=0.2, random_state=42)

### XGB Classifier Results
model = XGBClassifier()
model.fit(X_train, y_train)

#@title Input fields

h1_score = 0.115894 #@param {type:"number"}
h2_score = 0.273280 #@param {type:"number"}
h3_score = 0.269234 #@param {type:"number"}
h4_score = 0.011647 #@param {type:"number"}
h5_score = 0.0 #@param {type:"number"}
h6_score = 0.020552 #@param {type:"number"}
text_score = 0.381590 #@param {type:"number"}
title_score = 0.326399 #@param {type:"number"}
desc_score = 0.401051 #@param {type:"number"}
domain_score = 0.276322 #@param {type:"number"}
path_score = 0.090630 #@param {type:"number"}
alt_img_score = 0.342019 #@param {type:"number"}<jupyter_output><empty_output><jupyter_text>Predicted score * For XGB Classifier model prediction, use model name as 'model'* For Neural net model prediction, use model as 'ann_model'<jupyter_code>print('Artificial Neural Network Model results')
predict_risk(h1_score,h2_score, h3_score, h4_score, h5_score, h6_score, 
             text_score, title_score, desc_score, domain_score, path_score, 
             alt_img_score, ann_model)<jupyter_output>Artificial Neural Network Model results
WARNING:tensorflow:From <ipython-input-24-83b90802b859>:16: Sequential.predict_proba (from tensorflow.python.keras.engine.sequential) is deprecated and will be removed after 2021-01-01.
Instructions for updating:
Please use `model.predict()` instead.
The probability of above average is 23.49%
probability of below average is 62.72%
probability of average is 13.79%
{'fraction_non_alphanumeric': 0.1345672836418209, 'fraction_numerical': 0.046356511589127895, 'mean_word_length': 4.314576871953921, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '38205', 'license': 'Apache-2.0', 'n_tokens_mistral': 4605, 'n_tokens_neox': 4191, 'n_words': 917, 'path': 'multi_class_classification/scoring_model_multiclass.ipynb', 'repo_name': 'arjunjanamatti/small_ML_and_DL_projects', 'size': '11994'}