<jupyter_start><jupyter_text>model = keras.models.load_model('./PyOut/Models1/')model1 = keras.models.load_model('./PyOut/Models/')<jupyter_code>from tensorflow import keras

from keras import backend as K
from keras.callbacks import EarlyStopping


import pandas as pd
import datetime as dt
import numpy as np

import time
from sklearn.feature_extraction.text import CountVectorizer
import sklearn.metrics as metrics
import matplotlib.pyplot as plt
from sklearn.inspection import partial_dependence, plot_partial_dependence
from matplotlib.pyplot import figure
import shap
import tensorflow.keras.backend 

from nltk import ngrams
import nltk
from nltk.util import ngrams
from nltk.collocations import BigramCollocationFinder
from nltk.metrics import BigramAssocMeasures

import sys
#sys.path.insert(0, r"c:\users\ishaa\anaconda3\envs\tf_gpu\lib\site-packages")

from keras.preprocessing.text import Tokenizer
from keras import models
from keras import layers

import shap
import tensorflow.keras.backend 
from sklearn.preprocessing import LabelEncoder
from keras.optimizers import SGD
from keras import Sequential

df=pd.read_csv('./PyOut/session_level_ppcrsdall_0104.csv', sep=',', keep_default_na=False)
df.head()
len(df['CustomerID'].unique())
df_pc=(df.groupby(['page_count'])).agg(num_sessions=('SessionID', 'count')).reset_index()
total_sess = df.shape[0]

df_pc['perc_sess']=df_pc['num_sessions']/total_sess
df_pc.head(10)


session_with_flags=df.copy()

session_with_flags.loc[:,'twograms'] =  session_with_flags.loc[:,'twograms'].astype(str)
session_with_flags.loc[:,'CustomerID'] =  session_with_flags.loc[:,'CustomerID'].astype(str)
session_with_flags.loc[:,'Gender'] =  session_with_flags.loc[:,'Gender'].astype(str)
session_with_flags.loc[:,'Office_U'] =  session_with_flags.loc[:,'Office_U'].astype(str)
session_with_flags.loc[:,'Office_W'] =  session_with_flags.loc[:,'Office_W'].astype(str)
session_with_flags.loc[:,'Flag'] =  session_with_flags.loc[:,'Flag'].astype(str)

cols=session_with_flags.filter(regex=(".*timespent.*")).columns
# 
for c in cols:
    session_with_flags.loc[:,c] =  session_with_flags.loc[:,c].replace('',0.0)
    session_with_flags.loc[:,c] =  session_with_flags.loc[:,c].astype('float16')
#     del session_with_flags[c]
    print(c)
#     print(session_with_flags.loc[:,c].values == '').sum()


# In[39]:



encoder = LabelEncoder()
#  transfomed_label = encoder.fit_transform(["dog", "cat", "bird"])

# del (session_with_flags['SessionID'])
# session_with_flags.loc[:,'twograms'] =  encoder.fit_transform(session_with_flags.loc[:,'twograms'])
# session_with_flags.loc[:,'twograms'] =  session_with_flags.loc[:,'twograms'].astype('float16')

session_with_flags = session_with_flags.drop(['twograms'], axis=1)

# session_with_flags.loc[:,'CustomerID'] =  session_with_flags.loc[:,'CustomerID'].astype(str)
session_with_flags.loc[:,'CustomerID'] =  encoder.fit_transform(session_with_flags.loc[:,'CustomerID'])
# session_with_flags.loc[:,'CustomerID'] =  session_with_flags.loc[:,'CustomerID'].astype('float16')

session_with_flags.loc[:,'AgeCategory'] =  encoder.fit_transform(session_with_flags.loc[:,'AgeCategory'])
# session_with_flags.loc[:,'AgeCategory'] =  session_with_flags.loc[:,'AgeCategory'].astype('float16')

session_with_flags.loc[:,'Gender'] =  encoder.fit_transform(session_with_flags.loc[:,'Gender'])
# session_with_flags.loc[:,'Gender'] =  session_with_flags.loc[:,'Gender'].astype(int)

# session_with_flags.loc[:,'Office_U'] =  session_with_flags.loc[:,'Office_U'].astype(str)
session_with_flags.loc[:,'Office_U'] =  encoder.fit_transform(session_with_flags.loc[:,'Office_U'])
# session_with_flags.loc[:,'Office_U'] =  session_with_flags.loc[:,'Office_U'].astype('float16')

# session_with_flags.loc[:,'Office_W'] =  session_with_flags.loc[:,'Office_W'].astype(str)
session_with_flags.loc[:,'Office_W'] =  encoder.fit_transform(session_with_flags.loc[:,'Office_W'])
# session_with_flags.loc[:,'Office_W'] =  session_with_flags.loc[:,'Office_W'].astype('float16')

session_with_flags.loc[:,'Flag'] =  encoder.fit_transform(session_with_flags.loc[:,'Flag'])
# session_with_flags.loc[:,'Flag'] =  session_with_flags.loc[:,'Flag'].astype('float16')

print(session_with_flags.dtypes[session_with_flags.dtypes=='object'])


# In[40]:


session_with_flags.loc[:,'page_count'] =  session_with_flags.loc[:,'page_count'].astype('float16')
session_with_flags.loc[:,'session_time'] =  session_with_flags.loc[:,'page_count'].astype('float16')
session_with_flags.loc[:,'avg_time_per_page'] =  session_with_flags.loc[:,'avg_time_per_page'].astype('float16')

session_with_flags = session_with_flags.drop(['SessionID'], axis=1)



# 3 way split
msk = np.random.rand(len(session_with_flags)) < 0.8
train = session_with_flags[msk]
val_test = session_with_flags[~msk]

msk = np.random.rand(len(val_test)) < 0.5
validate = val_test[msk]
test = val_test[~msk]

train_target = train.loc[:,"Flag"]
validate_target = validate.loc[:,"Flag"]
test_target = test.loc[:,"Flag"]

print(len(train[train.Flag==1]),len(train)) #Sessions with flag =1 
print(len(validate[validate.Flag==1]),len(validate)) #Sessions with flag =1 
print(len(test[test.Flag==1]),len(test)) #Sessions with flag =1 

train_data = train.drop(['Flag'], axis=1)
validate_data = validate.drop(['Flag'], axis=1)
test_data = test.drop(['Flag'], axis=1)
# 2 way split - Train & Validate
msk = np.random.rand(len(session_with_flags)) < 0.7
train = session_with_flags[msk]
validate = session_with_flags[~msk]

# msk = np.random.rand(len(val_test)) < 0.5
# validate = val_test[msk]
# test = val_test[~msk]

train_target = train.loc[:,"Flag"]
validate_target = validate.loc[:,"Flag"]
# test_target = test.loc[:,"Flag"]

print(len(train[train.Flag==1]),len(train)) #Sessions with flag =1 
print(len(validate[validate.Flag==1]),len(validate)) #Sessions with flag =1 
# print(len(test[test.Flag==1]),len(test)) #Sessions with flag =1 

train_data = train.drop(['Flag'], axis=1)
validate_data = validate.drop(['Flag'], axis=1)
# test_data = test.drop(['Flag'], axis=1)
print(len(train[train.Flag==1])/len(train))
print(len(validate[validate.Flag==1])/len(validate))
# print(len(test[test.Flag==1])/len(test))
train_pc=(train.groupby(['page_count'])).agg(num_sessions=('CustomerID', 'count')).reset_index()
train_sess = train.shape[0]
train_pc['perc_sess']=train_pc['num_sessions']/train_sess
train_pc.head(10)
validate_pc=(validate.groupby(['page_count'])).agg(num_sessions=('CustomerID', 'count')).reset_index()
validate_sess = validate.shape[0]
validate_pc['perc_sess']=validate_pc['num_sessions']/validate_sess
validate_pc.head(10)
test_pc=(test.groupby(['page_count'])).agg(num_sessions=('CustomerID', 'count')).reset_index()
test_sess = test.shape[0]
test_pc['perc_sess']=test_pc['num_sessions']/test_sess
test_pc.head(10)

# train_data = np.reshape(train_data,( train_data.shape[0], 1,train_data.shape[1]))
train_data.shape
# 


def recall_m(y_true, y_pred): 
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) 
    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1))) 
    recall = true_positives / (possible_positives + K.epsilon()) 
    return recall

def precision_m(y_true, y_pred): 
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) 
    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1))) 
    precision = true_positives / (predicted_positives + K.epsilon()) 
    return precision

def f1_m(y_true, y_pred): 
    precision = precision_m(y_true, y_pred) 
    recall = recall_m(y_true, y_pred) 
    return 2*((precision*recall)/(precision+recall+K.epsilon()))

#https://datascience.stackexchange.com/questions/45165/how-to-get-accuracy-f1-precision-and-recall-for-a-keras-model
from keras import backend as K
def precision_threshold(threshold=0.5):
    def precision(y_true, y_pred):
        """Precision metric.
        Computes the precision over the whole batch using threshold_value.
        """
        threshold_value = threshold
        # Adaptation of the "round()" used before to get the predictions. Clipping to make sure that the predicted raw values are between 0 and 1.
        y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), threshold_value), K.floatx())
        # Compute the number of true positives. Rounding in prevention to make sure we have an integer.
        true_positives = K.round(K.sum(K.clip(y_true * y_pred, 0, 1)))
        # count the predicted positives
        predicted_positives = K.sum(y_pred)
        # Get the precision ratio
        precision_ratio = true_positives / (predicted_positives + K.epsilon())
        return precision_ratio
    return precision

def recall_threshold(threshold = 0.5):
    def recall(y_true, y_pred):
        """Recall metric.
        Computes the recall over the whole batch using threshold_value.
        """
        threshold_value = threshold
        # Adaptation of the "round()" used before to get the predictions. Clipping to make sure that the predicted raw values are between 0 and 1.
        y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), threshold_value), K.floatx())
        # Compute the number of true positives. Rounding in prevention to make sure we have an integer.
        true_positives = K.round(K.sum(K.clip(y_true * y_pred, 0, 1)))
        # Compute the number of positive targets.
        possible_positives = K.sum(K.clip(y_true, 0, 1))
        recall_ratio = true_positives / (possible_positives + K.epsilon())
        return recall_ratio
    return recall
# !pip install keras-self-attention
# from keras_self_attention import SeqSelfAttention


# TrainX= np.reshape(TrainX,(TrainX.shape[0], 1, TrainX.shape[1]))

# train_data = np.reshape(train_data,( 1, train_data.shape[0], train_data.shape[1]))

number_of_features = train_data.shape[1]

# # Start neural network
network = Sequential()

# # Add fully connected layer with a ReLU activation function
network.add(layers.InputLayer(input_shape=(number_of_features,)))
# network.add(layers.Dense(units=8, activation='relu', input_shape=(number_of_features,)))
# network.add(layers.Dense(units=250, activation='relu'))


# network.add(keras.layers.Reshape((1, number_of_features)))

# network.add(layers.recurrent.LSTM(100, implementation=2, kernel_initializer='glorot_uniform',return_sequences=False,activation="tanh", dropout=0.2, recurrent_dropout=0.0))
# # network.add(layers.recurrent.LSTM(100, return_sequences=False, dropout=0.2, input_shape=(8,number_of_features)))
# #https://github.com/fau-is/next-best-action/blob/master/nextbestaction/train.py
network.add(layers.normalization.BatchNormalization())
# # Add fully connected layer with a ReLU activation function
network.add(layers.Dropout(0.5))
network.add(layers.Dense(units=1000, activation='relu'))
network.add(layers.Dropout(0.5))
network.add(layers.Dense(units=1000, activation='relu'))
network.add(layers.Dropout(0.5))
network.add(layers.Dense(units=1000, activation='relu'))
network.add(layers.Dropout(0.5))
network.add(layers.Dense(units=1000, activation='relu'))

# network.add(layers.Dense(units=250, activation='relu'))
# network.add(layers.Dense(units=250, activation='relu'))
# network.add(SeqSelfAttention(attention_width=15,
#     attention_activation='sigmoid',
#     name='Attention',))


# # Add fully connected layer with a sigmoid activation function
network.add(layers.Dense(units=1, activation='sigmoid')) #output layer for binary classification

early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)
# optimizer=Adadelta(lr=.1.5)

network.compile(loss='binary_crossentropy', # Cross-entropy
                optimizer='adam', # Adam performs better than Adadelta and SGD
#                    optimizer = SGD(learning_rate=0.01, momentum=0.1, nesterov=False), 
#                 metrics=['accuracy']) # Accuracy performance metric
                metrics=[recall_m, precision_m,'acc',f1_m])
#                 metrics=[precision_threshold(0.1)])
#                 metrics=[precision_m])

# print("--- %s seconds for pre-processing  2---" % (time.time() - start_time))
start_time = time.time()

# validate_data = np.reshape(validate_data,( 1, validate_data.shape[0], validate_data.shape[1]))
class_weight = {0: 1.,
                1: 5.}

history = network.fit(train_data, # Features
                      train_target, # Target vector
                      epochs=100, # Number of epochs
                      verbose=1, # Print description after each epoch
                      batch_size=512, # Number of observations per batch
                      callbacks=early_stopping,
#                       class_weight=class_weight,
                      validation_data=(validate_data, validate_target)) # Data for evaluation

print("--- %s seconds for training---" % (time.time() - start_time))


# In[ ]:

network.save('./PyOut/Models_0104/'+str(time.time())+'/')
start_time = time.time()
# evaluate the model
loss, accuracy, f1_score, precision, recall = network.evaluate(test_data, test_target, verbose=1)
print("--- %s seconds for evaluation---" % (time.time() - start_time))<jupyter_output>Epoch 1/100
59/59 [==============================] - 5s 80ms/step - loss: 0.3793 - recall_m: 0.0912 - precision_m: 0.3618 - acc: 0.8875 - f1_m: 0.1367 - val_loss: 0.3469 - val_recall_m: 0.5211 - val_precision_m: 0.6665 - val_acc: 0.9201 - val_f1_m: 0.5795
Epoch 2/100
59/59 [==============================] - 4s 74ms/step - loss: 0.2741 - recall_m: 0.2900 - precision_m: 0.7307 - acc: 0.9055 - f1_m: 0.4083 - val_loss: 0.3464 - val_recall_m: 0.5350 - val_precision_m: 0.7083 - val_acc: 0.9258 - val_f1_m: 0.6043
Epoch 3/100
59/59 [==============================] - 4s 75ms/step - loss: 0.2637 - recall_m: 0.3078 - precision_m: 0.7661 - acc: 0.9091 - f1_m: 0.4337 - val_loss: 0.2981 - val_recall_m: 0.5225 - val_precision_m: 0.7486 - val_acc: 0.9286 - val_f1_m: 0.6103
Epoch 4/100
59/59 [==============================] - 4s 74ms/step - loss: 0.2602 - recall_m: 0.3402 - precision_m: 0.7659 - acc: 0.9117 - f1_m: 0.4653 - val_loss: 0.2600 - val_recall_m: 0.5065 - val_precision_m: 0.7700 - val_acc: 0.[...]<jupyter_text>loss, accuracy, f1_score, precision, recall = network.evaluate(test_data, test_target, verbose=1)loss, precision = network.evaluate(test_data, test_target, verbose=1)loss: 0.3370 - precision: 0.1129<jupyter_code>network.summary()
pred_cnn_val_y = network.predict([validate_data], batch_size=1024, verbose=1)
# for thresh in [.43,0.58,0.65,.72,0.87]:
for thresh in [.35,0.38,0.41,.44,0.47]:
    thresh = np.round(thresh, 3)
#     print("F1 score at threshold {0} is {1}".format(thresh, metrics.recall_score(validate_target, (pred_cnn_val_y>thresh).astype(int))))
    print("Score at threshold {0} is {1}".format(thresh, metrics.f1_score(validate_target, (pred_cnn_val_y>thresh).astype(int))))



pred_test_y = network.predict([test_data], batch_size=1024, verbose=1)
pred_test_y = (pred_test_y > 0.42).astype(int)

matrix = metrics.confusion_matrix(test_target, pred_test_y )

print(metrics.f1_score(test_target, pred_test_y ))
print(metrics.precision_score(test_target, pred_test_y ))
print(metrics.accuracy_score(test_target, pred_test_y ))
print(metrics.recall_score(test_target, pred_test_y ))

print(matrix)
print(matrix)

import matplotlib.pyplot as plt
# conf = sklearn.metrics.confusion_matrix(y_true, y_pred)
plt.imshow(matrix, cmap='binary', interpolation='None')
plt.show()
pred_test_y = network.predict([validate_data], batch_size=1024, verbose=1)
pred_test_y = (pred_test_y > 0.27).astype(int)

matrix = metrics.confusion_matrix(validate_target, pred_test_y )

print(metrics.f1_score(validate_target, pred_test_y ))
print(metrics.precision_score(validate_target, pred_test_y ))
print(metrics.accuracy_score(validate_target, pred_test_y ))
print(metrics.recall_score(validate_target, pred_test_y ))
print(matrix)
# plt.imshow(matrix, cmap='binary', interpolation='None')
# plt.show()<jupyter_output>13/13 [==============================] - 0s 30ms/step
0.6369737644905431
0.5793562708102109
0.9079802041447572
0.7073170731707317
[[10698   758]
 [  432  1044]]
{'fraction_non_alphanumeric': 0.15977361776229865, 'fraction_numerical': 0.037875489769264255, 'mean_word_length': 5.4423076923076925, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '522654', 'license': 'MIT', 'n_tokens_mistral': 6083, 'n_tokens_neox': 5611, 'n_words': 957, 'path': 'SequentialDataModelling/UWVTrain.ipynb', 'repo_name': 'ishaanrd/ExplainablePrescriptiveAnalytics', 'size': '16079'}