<jupyter_start><jupyter_text>TokenCheck<jupyter_code>#Import Libraries
import numpy as np
import pandas as pd
from keras.layers import Dense, Embedding, LSTM, SpatialDropout1D, Dropout
from keras.models import Sequential
from sklearn.feature_extraction.text import CountVectorizer
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from sklearn.model_selection import train_test_split
from keras.utils.np_utils import to_categorical
from keras.callbacks import EarlyStopping
import matplotlib.pyplot as plt
%matplotlib inline
from pandas.plotting import table<jupyter_output><empty_output><jupyter_text>Defining functions<jupyter_code>def label(df):    
    # label data
    df['LABEL'] = 0
    df.loc[df['CATEGORY'] == 0, 'LABEL'] = 0
    df.loc[df['CATEGORY'] != 0, 'LABEL'] = 1
    
def preprocess(df):
    n_most_common_words = 8000 #8000
    max_len = 142

    
    tokenizer = Tokenizer(num_words=n_most_common_words, lower=False)
    tokenizer.fit_on_texts(df['OPCODE'].values)
    sequences = tokenizer.texts_to_sequences(df['OPCODE'].values)
    word_index = tokenizer.word_index
    X = pad_sequences(sequences, maxlen=max_len)
    return X

def dftoXY(df):
    X_test = preprocess(df)
    label(df)
    print(pd.value_counts(df['LABEL']))
    y_test = to_categorical(df['LABEL'], num_classes=2)
    return X_test, y_test    

def XandY(posdf, negdf):
    dfset = pd.concat([posdf, negdf])
    dfset = dfset.sample(frac=1, random_state=39, replace=False)
    dfset['LABEL'] = 0
    dfset.loc[dfset['CATEGORY'] == 0, 'LABEL'] = 0
    dfset.loc[dfset['CATEGORY'] != 0, 'LABEL'] = 1
    X, y = dftoXY(dfset)

    print('Shape of X: {}'.format(X.shape))

    y_labels = np.expand_dims(np.array(np.argmax(y, axis=1)), axis=1)
    print('Shape of y: {}'.format(y_labels.shape))

    return X, y_labels<jupyter_output><empty_output><jupyter_text>Loading the dataset<jupyter_code>data = pd.read_csv("out.csv")
data.head()<jupyter_output><empty_output><jupyter_text>Checking the target classes<jupyter_code>pd.value_counts(data['CATEGORY'])<jupyter_output><empty_output><jupyter_text>Plot unbalanced<jupyter_code>fig = plt.figure()
count_classes = pd.value_counts(data['CATEGORY'], sort = True).sort_index()
count_classes.plot(kind = 'bar')
plt.title("Vulnerability Class Histogram")
plt.xlabel("Class")
plt.ylabel("Frequency")
fig.set_size_inches(8, 8)
plt.show()
# fig.savefig('./figures/clean_train_unbalanced_distr.png', dpi=300) #, bbox_inches='tight'<jupyter_output><empty_output><jupyter_text>Dividing data into vulnerable and non-vulnerable<jupyter_code>shuffled = data

n = shuffled[shuffled['CATEGORY'] == 0] # no vulnerabilities
s = shuffled[shuffled['CATEGORY'] == 1] # suicidal<jupyter_output><empty_output><jupyter_text>Plot two class unbalanced<jupyter_code>concated = pd.concat([n,s], ignore_index=True)

concated = concated.reindex(np.random.permutation(concated.index))
concated['LABEL'] = 0

concated.loc[concated['CATEGORY'] == 0, 'LABEL'] = 0
concated.loc[concated['CATEGORY'] != 0, 'LABEL'] = 1

import seaborn as sns
fig = plt.figure()
colors = ["#A1CAF1", "#E30022"]

sns.countplot('LABEL', data=concated, palette=colors)
plt.title('Class Distributions \n (0 = Not vulnerable || 1 = Vulnerable)', fontsize=14)
fig.set_size_inches(8, 8)<jupyter_output>E:\Softwares\anaconda\envs\my-rdkit-env\lib\site-packages\seaborn\_decorators.py:43: FutureWarning: Pass the following variable as a keyword arg: x. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  FutureWarning<jupyter_text>Dividing data into train, validation, and test<jupyter_code>positives = pd.concat([s])
positives_shuf = positives.sample(frac=1, random_state=39, replace=False)
print(len(positives_shuf))

proportion_train = 0.64 #0.7 
proportion_val = 0.16 #0.1 
proportion_test = 0.2 #0.20

num_pos_train = round(len(positives_shuf) * proportion_train)
num_pos_val = round(len(positives_shuf) * proportion_val)

pos_train = positives_shuf.iloc[0:num_pos_train] 
pos_val = positives_shuf.iloc[num_pos_train:(num_pos_train+num_pos_val)]
pos_test = positives_shuf.iloc[(num_pos_train+num_pos_val):]

print(len(pos_train))
print(len(pos_val))
print(len(pos_test))
n_shuf = n.sample(frac=1, random_state=39, replace=False)
print(len(n_shuf))

num_neg_train = round(100000*0.64)

num_neg_val = round(((num_neg_train)/proportion_train)*proportion_val)
num_neg_test = round(((num_neg_train)/proportion_train)*proportion_test)

neg_train = n_shuf.iloc[0:num_neg_train]
neg_val = n_shuf.iloc[num_neg_train:(num_neg_train+num_neg_val)]
neg_test = n_shuf.iloc[(num_neg_train+num_neg_val):(num_neg_train+num_neg_val+num_neg_test)]


print(len(neg_train))
print(len(neg_val))
print(len(neg_test))<jupyter_output>155667
64000
16000
20000<jupyter_text>Resampling samples<jupyter_code>df_train = pd.concat([pos_train, neg_train])
df_train = df_train.sample(frac=1, random_state=39, replace=False)

df_train['LABEL'] = 0


df_train.loc[df_train['CATEGORY'] == 0, 'LABEL'] = 0
df_train.loc[df_train['CATEGORY'] != 1, 'LABEL'] = 1


X_train, y_train = dftoXY(df_train)

print('Shape of X: {}'.format(X_train.shape))


ytrain_labels = np.expand_dims(np.array(np.argmax(y_train, axis=1)), axis=1)
print('Shape of y: {}'.format(ytrain_labels.shape))
df_val = pd.concat([pos_val, neg_val])
df_val = df_val.sample(frac=1, random_state=39, replace=False)

df_val['LABEL'] = 0


df_val.loc[df_val['CATEGORY'] == '1 0 0 0', 'LABEL'] = 0
df_val.loc[df_val['CATEGORY'] != '1 0 0 0', 'LABEL'] = 1


X_val, y_val = dftoXY(df_val)

print('Shape of X: {}'.format(X_val.shape))


yval_labels = np.expand_dims(np.array(np.argmax(y_val, axis=1)), axis=1)
print('Shape of y: {}'.format(yval_labels.shape))
df_test = pd.concat([pos_test, neg_test])
df_test = df_test.sample(frac=1, random_state=39, replace=False)

df_test['LABEL'] = 0


df_test.loc[df_test['CATEGORY'] == '1 0 0 0', 'LABEL'] = 0
df_test.loc[df_test['CATEGORY'] != '1 0 0 0', 'LABEL'] = 1


X_test, y_test = dftoXY(df_test)

print('Shape of X: {}'.format(X_test.shape))


ytest_labels = np.expand_dims(np.array(np.argmax(y_test, axis=1)), axis=1)
print('Shape of y: {}'.format(ytest_labels.shape))<jupyter_output><empty_output><jupyter_text>Resampling samples<jupyter_code>X_train, ytrain_labels = XandY(pos_train, neg_train)
X_val, yval_labels = XandY(pos_val, neg_val)
X_test, ytest_labels = XandY(pos_test, neg_test)
X_train
print("Number transactions X_train dataset: ", X_train.shape)
print("Number transactions y_train dataset: ", ytrain_labels.shape)
print("Number transactions X_val dataset: ", X_val.shape)
print("Number transactions y_val dataset: ", yval_labels.shape)
print("Number transactions X_test dataset: ", X_test.shape)
print("Number transactions y_test dataset: ", ytest_labels.shape)

print("Before OverSampling, counts of label '1': {}".format(sum(ytrain_labels==1)))
print("Before OverSampling, counts of label '0': {}".format(sum(ytrain_labels==0)))
print("Before OverSampling, counts of label '1': {}".format(sum(yval_labels==1)))
print("Before OverSampling, counts of label '0': {}".format(sum(yval_labels==0)))
print("Before OverSampling, counts of label '1': {}".format(sum(ytest_labels==1)))
print("Before OverSampling, counts of label '0': {} \n".format(sum(ytest_labels==0)))
from imblearn.over_sampling import SMOTE

sm = SMOTE(random_state=39)
X_train_res, y_train_res = sm.fit_sample(X_train, ytrain_labels.ravel())
X_val_res, y_val_res = sm.fit_sample(X_val, yval_labels.ravel())
X_test_res, y_test_res = sm.fit_sample(X_test, ytest_labels.ravel())
print('After OverSampling, the shape of train_X: {}'.format(X_train_res.shape))
print('After OverSampling, the shape of train_y: {}'.format(y_train_res.shape))
print('After OverSampling, the shape of val_X: {}'.format(X_val_res.shape))
print('After OverSampling, the shape of val_y: {}'.format(y_val_res.shape))
print('After OverSampling, the shape of test_X: {}'.format(X_test_res.shape))
print('After OverSampling, the shape of test_y: {} \n'.format(y_test_res.shape))

print("After OverSampling, counts of train label '1': {}".format(sum(y_train_res==1)))
print("After OverSampling, counts of train label '0': {}".format(sum(y_train_res==0)))
print("After OverSampling, counts of val label '1': {}".format(sum(y_val_res==1)))
print("After OverSampling, counts of val label '0': {}".format(sum(y_val_res==0)))
print("After OverSampling, counts of test label '1': {}".format(sum(y_test_res==1)))
print("After OverSampling, counts of test label '0': {}".format(sum(y_test_res==0)))<jupyter_output>After OverSampling, the shape of train_X: (128000, 142)
After OverSampling, the shape of train_y: (128000,)
After OverSampling, the shape of val_X: (32000, 142)
After OverSampling, the shape of val_y: (32000,)
After OverSampling, the shape of test_X: (40000, 142)
After OverSampling, the shape of test_y: (40000,) 

After OverSampling, counts of train label '1': 64000
After OverSampling, counts of train label '0': 64000
After OverSampling, counts of val label '1': 16000
After OverSampling, counts of val label '0': 16000
After OverSampling, counts of test label '1': 20000
After OverSampling, counts of test label '0': 20000<jupyter_text>Plot two class balanced<jupyter_code>entire_dataset = np.concatenate((y_train_res,y_val_res,y_test_res))
import seaborn as sns
fig = plt.figure()
colors = ["#A1CAF1", "#E30022"]

sns.countplot(0, data=pd.DataFrame(entire_dataset), palette=colors)
plt.title('Class Distributions \n (0 = Not vulnerable || 1 = Vulnerable)', fontsize=14)
fig.set_size_inches(8, 8)<jupyter_output>E:\Softwares\anaconda\envs\my-rdkit-env\lib\site-packages\seaborn\_decorators.py:43: FutureWarning: Pass the following variable as a keyword arg: x. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  FutureWarning<jupyter_text>Multiclass LSTM Model<jupyter_code>epochs = 150
emb_dim = 128
batch_size = 256 

import keras
keras.backend.clear_session()
ytrainres_cat = to_categorical(y_train_res, num_classes=2)
yvalres_cat = to_categorical(y_val_res, num_classes=2)
ytestres_cat = to_categorical(y_test_res, num_classes=2)

print((X_train_res.shape, ytrainres_cat.shape, X_val_res.shape, yvalres_cat.shape, X_test_res.shape, ytestres_cat.shape))
n_most_common_words = 1000
model = Sequential()
model.add(Embedding(n_most_common_words, emb_dim, input_length=X_train_res.shape[1]))
model.add(SpatialDropout1D(0.6))
model.add(LSTM(64, dropout=0.6, recurrent_dropout=0.6))

model.add(Dense(2, activation='sigmoid'))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc'])
print(model.summary())

import time
start_time = time.time()

history = model.fit(X_train_res, ytrainres_cat, epochs=epochs, batch_size=batch_size, validation_split=0.0, validation_data=(X_val_res, yvalres_cat),callbacks=[EarlyStopping(monitor='loss',patience=7, min_delta=0.0001)])
end_time = time.time()
print('Time taken for training: ', end_time-start_time)<jupyter_output>Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding (Embedding)        (None, 142, 128)          128000    
_________________________________________________________________
spatial_dropout1d (SpatialDr (None, 142, 128)          0         
_________________________________________________________________
lstm (LSTM)                  (None, 64)                49408     
_________________________________________________________________
dense (Dense)                (None, 2)                 130       
=================================================================
Total params: 177,538
Trainable params: 177,538
Non-trainable params: 0
_________________________________________________________________
None
Epoch 1/150
500/500 [==============================] - 217s 434ms/step - loss: 0.4162 - acc: 0.8109 - val_loss:[...]<jupyter_text>Test and evaluate model accuracy<jupyter_code>accr = model.evaluate(X_test_res, ytestres_cat)
print('Test set\n  Loss: {:0.4f}\n  Accuracy: {:0.4f}'.format(accr[0],accr[1]))
y_pred = model.predict_classes(X_test_res, batch_size=32, verbose=1)
ytest_true = y_test_res
from sklearn.metrics import roc_curve
fpr_keras, tpr_keras, thresholds_keras = roc_curve(ytest_true, y_pred)
from sklearn.metrics import auc
auc_keras = auc(fpr_keras, tpr_keras)
plt.figure(1)
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(fpr_keras, tpr_keras, label=' Area = {:.3f}'.format(auc_keras))
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('ROC curve')
plt.legend(loc='best')
plt.show()
# Zoom in view of the upper left corner.
plt.figure(2)
plt.xlim(0, 0.2)
plt.ylim(0.8, 1)
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(fpr_keras, tpr_keras, label='Keras (area = {:.3f})'.format(auc_keras))
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('ROC curve (zoomed in at top left)')
plt.legend(loc='best')
plt.show()
from sklearn.metrics import average_precision_score

average_precision = average_precision_score(ytest_true, y_pred)
print('Average Precision Score: {:0.4f}\n'.format(average_precision))
from sklearn.metrics import precision_score, \
    recall_score, confusion_matrix, classification_report, \
    accuracy_score, f1_score

print('Accuracy:', accuracy_score(ytest_true, y_pred))
print('Recall:', recall_score(ytest_true, y_pred))
print('Precision:', precision_score(ytest_true, y_pred))
print('F1 score:', f1_score(ytest_true, y_pred))
print('\n clasification report:\n', classification_report(ytest_true, y_pred))
print('\n confusion matrix:\n',confusion_matrix(ytest_true, y_pred))
import seaborn as sn
df_cm = pd.DataFrame(confusion_matrix(ytest_true, y_pred), columns=np.unique(ytest_true), index = np.unique(ytest_true))
df_cm.index.name = 'Actual'
df_cm.columns.name = 'Predicted'
plt.figure(figsize = (10,7))
sn.set(font_scale=1.4)#for label size
sn.heatmap(df_cm, cmap="Blues", annot=True,annot_kws={"size": 16}, fmt= 'g')# font size<jupyter_output><empty_output><jupyter_text>Plot Performance<jupyter_code>acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(1, len(acc) + 1)

fig = plt.figure()
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()

# fig.savefig('clean_train_acc.png',dpi=300)
fig = plt.figure()
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()
# fig.savefig('clean_train_loss.png',dpi=300)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15016884062768987, 'fraction_numerical': 0.028272528636694695, 'mean_word_length': 6.838090295796575, 'pattern_counts': {'":': 1, '<': 52, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '518271', 'license': 'MIT', 'n_tokens_mistral': 5846, 'n_tokens_neox': 5478, 'n_words': 1099, 'path': 'Notebooks/Model.ipynb', 'repo_name': 'subhasishgoswami/TokenCheck', 'size': '15103'}