<jupyter_start><jupyter_text>Implementation of Anomaly detection using AutoencodersDataset used here is Credit Card Fraud Detection from Kaggle. Import required libraries<jupyter_code>import pandas as pd
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler,normalize, MinMaxScaler
from sklearn.metrics import confusion_matrix, recall_score, accuracy_score, precision_score

from keras import backend as K
print("-------------------------------------------")
print("GPU available: ", tf.test.is_gpu_available())
print("Keras backend: ", K.backend())
print("-------------------------------------------")

# Load layers from keras
from keras.layers import Dense, Input, Concatenate, Flatten, BatchNormalization, Dropout, LeakyReLU
from keras.models import Sequential, Model
from keras.losses import binary_crossentropy
from Disco_tf import distance_corr
from keras.optimizers import Adam
from sklearn.metrics import roc_auc_score
RANDOM_SEED = 2021 
TEST_PCT = 0.3
LABELS = ["Normal","Fraud"]
# define new loss with distance decorrelation
def decorr(var_1, var_2, weights,kappa):

    def loss(y_true, y_pred):
        #return binary_crossentropy(y_true, y_pred) + distance_corr(var_1, var_2, weights)
        #return distance_corr(var_1, var_2, weights)
        return binary_crossentropy(y_true, y_pred) + kappa * distance_corr(var_1, var_2, weights)
        #return binary_crossentropy(y_true, y_pred)

    return loss<jupyter_output><empty_output><jupyter_text>Read the dataset<jupyter_code>##########################################################
# ------------------------------------------------------ #
# ----------------------- LOADING ---------------------- #
# ------------------------------------------------------ #
##########################################################
# Firstly the model loads the background and signal data, 
# then it removes the attributes first string line, which 
# are the column names, in order to avoid NaN values in 
# the array.

print('         ==== Commencing Initiation ====\n')

### Background
b_name='Input_Background_1.csv'
background = np.genfromtxt(b_name, delimiter=',')
background = background[1:,:]
print("     .Background Loaded..." )
print("     .Background shape: {}".format(background.shape))

### Signal
s_name='Input_Signal_1.csv'
signal = np.genfromtxt(s_name, delimiter=',')
signal = signal[1:,:]
print("     .Signal Loaded...")
print("     .Signal shape: {}\n".format(signal.shape))
### Selecting the most pertinent attributes

pertinent_features = [2,3,4,6,10,11,12,14,15,16,18,19,20]

background = background[:,pertinent_features]
signal = signal[:,pertinent_features]
##########################################################
# ------------------------------------------------------ #
# --------------------- INITIATION --------------------- #
# ------------------------------------------------------ #
##########################################################

# Number of events
total = 1000

# Percentage of background samples on the testing phase
background_percent = 0.99

# Percentage of samples on the training phase
test_size = 0.3
print('\n          ==== Initiation Complete ====\n')
print('=*='*17 )
print('      ==== Commencing Data Processing ====')

# Percentage of background samples to divide the data-set
dat_set_percent = total/len(background)

# Reducing background samples
_,reduced_background = train_test_split(background, test_size=dat_set_percent)

# Iserting the correct number of signal in streaming

n_signal_samples = int(len(reduced_background)*(1-background_percent))

_,reduced_background = train_test_split(reduced_background, test_size=background_percent)

_,reduced_signal = train_test_split(signal, test_size=n_signal_samples/len(signal))

# Concatenating Signal and the Background sub-sets

data = np.vstack((reduced_background,reduced_signal))

print(".formatted data shape: {}".format(data.shape))
print(".Background shape: {}".format(reduced_background.shape))
print(".Signal shape: {}".format(reduced_signal.shape))

# Normalize Data
print('.Normalizing Data')
data = normalize(data,norm='max',axis=0)

# Creating Labels
print('.Creating Labels')

labels =np.ones((len(data)))
labels[:len(reduced_background)] = 0
attributes = np.array(["px1","py1","pz1","E1","eta1","phi1","pt1","px2","py2","pz2","E2","eta2",
            "phi2","pt2","Delta_R","M12","MET","S","C","HT","A",'Class'])

dataset = pd.DataFrame(np.hstack((data,y.reshape(-1,1))),columns = attributes)<jupyter_output><empty_output><jupyter_text>Exploratory Data Analysis<jupyter_code>dataset.head()
#check for any  nullvalues 
print("Any nulls in the dataset ",dataset.isnull().values.any() )
print('-------')
print("No. of unique labels ", len(dataset['Class'].unique()))
print("Label values ",dataset.Class.unique())
#0 is for normal credit card transaction
#1 is for fraudulent credit card transaction
print('-------')
print("Break down of the Normal and Fraud Transactions")
print(pd.value_counts(dataset['Class'], sort = True) )<jupyter_output>Any nulls in the dataset  False
-------
No. of unique labels  2
Label values  [0. 1.]
-------
Break down of the Normal and Fraud Transactions
0.0    990
1.0     10
Name: Class, dtype: int64<jupyter_text>Visualize the datasetplotting the number of normal and fraud transactions in the dataset.<jupyter_code>#Visualizing the imbalanced dataset
count_classes = pd.value_counts(dataset['Class'], sort = True)
count_classes.plot(kind = 'bar', rot=0)
plt.xticks(range(len(dataset['Class'].unique())), dataset.Class.unique())
plt.title("Frequency by observation number")
plt.xlabel("Class")
plt.ylabel("Number of Observations");<jupyter_output><empty_output><jupyter_text>Visualizing the amount for normal and fraud transactions. Deixar para uma próxima, colocar bayes aqui<jupyter_code>#Save the normal and fradulent transactions in separate dataframe
normal_dataset = dataset[dataset.Class == 0] 
fraud_dataset = dataset[dataset.Class == 1]
#Visualize transactionamounts for normal and fraudulent transactions
bins = np.linspace(200, 2500, 100)
plt.hist(normal_dataset.Amount, bins=bins, alpha=1, density=True, label='Normal')
plt.hist(fraud_dataset.Amount, bins=bins, alpha=0.5, density=True, label='Fraud')
plt.legend(loc='upper right')
plt.title("Transaction amount vs Percentage of transactions")
plt.xlabel("Transaction amount (USD)")
plt.ylabel("Percentage of transactions");
plt.show()<jupyter_output><empty_output><jupyter_text>Create train and test datasetChecking on the dataset The last column in the dataset is our target variable.<jupyter_code>def probability(df):
    mu = np.mean(df, axis=0)
    variance = np.mean((df - mu)**2, axis=0)
    var_dia = np.diag(variance)
    k = len(mu)
    X = df - mu
    p = 1/((2*np.pi)**(k/2)*(np.linalg.det(var_dia)**0.5))* np.exp(-0.5* np.sum(X @ np.linalg.pinv(var_dia) * X,axis=1))
    return p
raw_data = dataset.values
# The other data points are the electrocadriogram data
data = raw_data[:, 0:-1]
train_data, test_data, train_labels, test_labels = train_test_split(
    data, labels, test_size=0.3, random_state=2021 ,stratify = labels)

p_train = probability(train_data)
p_test = probability(test_data)<jupyter_output><empty_output><jupyter_text>> Use only normal transactions to train the Autoencoder.Normal data has a value of 0 in the target variable. Using the target variable to create a normal and fraud dataset.<jupyter_code>train_labels = train_labels.astype(bool)
test_labels = test_labels.astype(bool)
#creating normal and fraud datasets
normal_train_data = train_data[~train_labels]
normal_p_train = p_train[~train_labels]

normal_test_data = test_data[~test_labels]
normal_p_test = p_test[~test_labels]

fraud_train_data = train_data[train_labels]
fraud_p_train = p_train[train_labels]

fraud_test_data = test_data[test_labels]
fraud_p_test = p_test[test_labels]
print(" No. of records in Fraud Train Data=",len(fraud_train_data))
print(" No. of records in Normal Train data=",len(normal_train_data))
print(" No. of records in Fraud Test Data=",len(fraud_test_data))
print(" No. of records in Normal Test data=",len(normal_test_data))<jupyter_output>No. of records in Fraud Train Data= 7
 No. of records in Normal Train data= 693
 No. of records in Fraud Test Data= 3
 No. of records in Normal Test data= 297<jupyter_text>Set the training parameter values<jupyter_code>nb_epoch = 150
batch_size = 200
input_dim = normal_train_data.shape[1]
encoding_dim = 6
hidden_dim_1 = int(encoding_dim / 2) #
hidden_dim_2=10  
learning_rate = 1e-7<jupyter_output><empty_output><jupyter_text>Create the AutoencoderThe architecture of the autoencoder is shown below.![esquema](autoencoder.png)<jupyter_code>#input Layer
input_layer = tf.keras.layers.Input(shape=(input_dim, ))
#Encoder
encoder = tf.keras.layers.Dense(encoding_dim, activation="tanh",                                
activity_regularizer=tf.keras.regularizers.l2(learning_rate))(input_layer)
encoder=tf.keras.layers.Dropout(0.2)(encoder)
encoder = tf.keras.layers.Dense(hidden_dim_1, activation='relu')(encoder)
encoder = tf.keras.layers.Dense(hidden_dim_2, activation=tf.nn.leaky_relu)(encoder)
# Decoder
decoder = tf.keras.layers.Dense(hidden_dim_1, activation='relu')(encoder)
decoder=tf.keras.layers.Dropout(0.2)(decoder)
decoder = tf.keras.layers.Dense(encoding_dim, activation='relu')(decoder)
decoder = tf.keras.layers.Dense(input_dim, activation='tanh')(decoder)
#Autoencoder
autoencoder = tf.keras.Model(inputs=input_layer, outputs=decoder)
autoencoder.summary()<jupyter_output>Model: "model_14"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_19 (InputLayer)        [(None, 21)]              0         
_________________________________________________________________
dense_84 (Dense)             (None, 6)                 132       
_________________________________________________________________
dropout_28 (Dropout)         (None, 6)                 0         
_________________________________________________________________
dense_85 (Dense)             (None, 3)                 21        
_________________________________________________________________
dense_86 (Dense)             (None, 10)                40        
_________________________________________________________________
dense_87 (Dense)             (None, 3)                 33        
__________________________________________________________[...]<jupyter_text>Define the callbacks for checkpoints and early stopping<jupyter_code>cp = tf.keras.callbacks.ModelCheckpoint(filepath="autoencoder_fraud.h5",
                               mode='min', monitor='val_loss', verbose=2, save_best_only=True)
# define our early stopping
early_stop = tf.keras.callbacks.EarlyStopping(
    monitor='val_loss',
    min_delta=0.0001,
    patience=10,
    verbose=1, 
    mode='min',
    restore_best_weights=True
)<jupyter_output><empty_output><jupyter_text>Compile the Autoencoder<jupyter_code>autoencoder.compile(metrics=['accuracy'],
                    loss='mean_squared_error',
                    optimizer='adam')<jupyter_output><empty_output><jupyter_text>Train the Autoencoder<jupyter_code>history = autoencoder.fit(normal_train_data, normal_train_data,
                    epochs=nb_epoch,
                    batch_size=batch_size,
                    shuffle=True,
                    validation_data=(test_data, test_data),
                    verbose=1,
                    callbacks=[cp, early_stop]
                    ).history<jupyter_output>Epoch 1/150
4/4 [==============================] - 2s 284ms/step - loss: 0.1286 - accuracy: 0.0857 - val_loss: 0.1266 - val_accuracy: 0.2067

Epoch 00001: val_loss improved from inf to 0.12665, saving model to autoencoder_fraud.h5
Epoch 2/150
4/4 [==============================] - 0s 34ms/step - loss: 0.1263 - accuracy: 0.1943 - val_loss: 0.1244 - val_accuracy: 0.1967

Epoch 00002: val_loss improved from 0.12665 to 0.12442, saving model to autoencoder_fraud.h5
Epoch 3/150
4/4 [==============================] - 0s 32ms/step - loss: 0.1223 - accuracy: 0.1942 - val_loss: 0.1227 - val_accuracy: 0.2067

Epoch 00003: val_loss improved from 0.12442 to 0.12272, saving model to autoencoder_fraud.h5
Epoch 4/150
4/4 [==============================] - 0s 30ms/step - loss: 0.1204 - accuracy: 0.1937 - val_loss: 0.1213 - val_accuracy: 0.2400

Epoch 00004: val_loss improved from 0.12272 to 0.12130, saving model to autoencoder_fraud.h5
Epoch 5/150
4/4 [==============================] - 0s 39ms/step - l[...]<jupyter_text>Plot training and test loss<jupyter_code>plt.plot(history['loss'], linewidth=2, label='Train')
plt.plot(history['val_loss'], linewidth=2, label='Test')
plt.legend(loc='upper right')
plt.title('Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
#plt.ylim(ymin=0.70,ymax=1)
plt.show()<jupyter_output><empty_output><jupyter_text>Detect Anomalies on test data> Anomalies are data points where the reconstruction loss is higherTo calculate the reconstruction loss on test data, predict the test data and calculate the mean square error between the test data and the reconstructed test data.<jupyter_code>test_x_predictions = autoencoder.predict(test_data)
mse = np.mean(np.power(test_data - test_x_predictions, 2), axis=1)
error_df = pd.DataFrame({'Reconstruction_error': mse,
                        'True_class': test_labels})<jupyter_output><empty_output><jupyter_text>Plotting the test data points and their respective reconstruction error sets a threshold value to visualize if the threshold value needs to be adjusted.<jupyter_code>threshold_fixed = 0.04
groups = error_df.groupby('True_class')
fig, ax = plt.subplots()
for name, group in groups:
    ax.plot(group.index, group.Reconstruction_error, marker='o', ms=3.5, linestyle='',
            label= "Fraud" if name == 1 else "Normal")
ax.hlines(threshold_fixed, ax.get_xlim()[0], ax.get_xlim()[1], colors="r", zorder=100, label='Threshold')
ax.legend()
plt.title("Reconstruction error for normal and fraud data")
plt.ylabel("Reconstruction error")
plt.xlabel("Data point index")
plt.show()<jupyter_output><empty_output><jupyter_text>Detect anomalies as points where the reconstruction loss is greater than a fixed threshold. Here we see that a value of 52 for the threshold will be good. Evaluating the performance of the anomaly detection<jupyter_code>threshold_fixed =0.04
pred_y = [1 if e > threshold_fixed else 0 for e in error_df.Reconstruction_error.values]
error_df['pred'] =pred_y
conf_matrix = confusion_matrix(error_df.True_class, pred_y)
plt.figure(figsize=(4, 4))
sns.heatmap(conf_matrix, xticklabels=LABELS, yticklabels=LABELS, annot=True, fmt="d");
plt.title("Confusion matrix")
plt.ylabel('True class')
plt.xlabel('Predicted class')
plt.show()
# print Accuracy, precision and recall
print(" Accuracy: ",accuracy_score(error_df['True_class'], error_df['pred']))
print(" Recall: ",recall_score(error_df['True_class'], error_df['pred']))
print(" Precision: ",precision_score(error_df['True_class'], error_df['pred']))<jupyter_output><empty_output><jupyter_text>As our dataset is highly imbalanced, we see a high accuracy but a low recall and precision.Things to further improve precision and recall would add more relevant features, different architecture for autoencoder, different hyperparameters, or a different algorithm. Conclusion:Autoencoder can be used as an anomaly detection algorithm when we have an unbalanced dataset where we have a lot of good examples and only a few anomalies. Autoencoders are trained to minimize reconstruction error. When we train the autoencoders on normal data or good data, we can hypothesize that the anomalies will have higher reconstruction errors than the good or normal data.<jupyter_code># Setup network
# make inputs
jets = Input(shape=X_train[0].shape[1:])
f_jets = Flatten()(jets)
leps = Input(shape=X_train[1].shape[1:])
f_leps = Flatten()(leps)
met = Input(shape=X_train[2].shape[1:])
i = Concatenate(axis=-1)([f_jets, f_leps, met])
sample_weights = Input(shape=(1,))
#setup trainable layers
d1 = get_block(i, 1024)
d2 = get_block(d1, 1024)
d3 = get_block(d2, 512)
d4 = get_block(d3, 256)
d5 = get_block(d4, 128)
o = Dense(1, activation="sigmoid")(d5)

model = Model(inputs=[jets,leps,met, sample_weights], outputs=o)
model.summary()

# Compile model
opt = Adam(lr=0.001)
model.compile(optimizer=opt, loss=decorr(jets[:,0,0], o[:,0], sample_weights[:,0]))
#model.compile(optimizer=opt, loss="binary_crossentropy")

# Train model
model.fit(x=X_train, y=y_train, epochs=20, batch_size=10000, validation_split=0.1)

# Evaluate model
y_train_predict = model.predict(X_train, batch_size=10000)
y_test_predict = model.predict(X_test, batch_size=10000)
auc_train = roc_auc_score(y_train, y_train_predict)
auc_test = roc_auc_score(y_test, y_test_predict)
print("area under ROC curve (train sample): ", auc_train)
print("area under ROC curve (test sample): ", auc_test)

# plot correlation
x = X_test[0][:,0,0]
y = y_test_predict[:,0]
corr = np.corrcoef(x, y)
print("correlation ", corr[0][1])

fig = plt.figure()
plt.scatter(x,y)
plt.xlabel("leading jet pt")
plt.ylabel("classifier output")

plt.show()
#fig.savefig("/work/creissel/TTH/sw/CMSSW_9_4_9/src/TTH/DNN/DisCo/corr.png")
#input Layer
input_layer = Input(shape=(input_dim, ))
sample_weights = Input(shape=(1,))
#Encoder
encoder = Dense(encoding_dim, activation="sigmoid",                                
activity_regularizer=tf.keras.regularizers.l2(learning_rate))(input_layer)
encoder=tf.keras.layers.Dropout(0.2)(encoder)
encoder = tf.keras.layers.Dense(hidden_dim_1, activation='relu')(encoder)
encoder = tf.keras.layers.Dense(hidden_dim_2, activation=tf.nn.leaky_relu)(encoder)
# Decoder
decoder = tf.keras.layers.Dense(hidden_dim_1, activation='relu')(encoder)
decoder=tf.keras.layers.Dropout(0.2)(decoder)
decoder = tf.keras.layers.Dense(encoding_dim, activation='relu')(decoder)
decoder = tf.keras.layers.Dense(input_dim, activation='sigmoid')(decoder)
#Autoencoder
autoencoder = tf.keras.Model(inputs=[input_layer, sample_weights], outputs=decoder)
autoencoder.summary()
cp = tf.keras.callbacks.ModelCheckpoint(filepath="autoencoder_fraud.h5",
                               mode='min', monitor='val_loss', verbose=2, save_best_only=True)
# define our early stopping
early_stop = tf.keras.callbacks.EarlyStopping(
    monitor='val_loss',
    min_delta=0.0001,
    patience=10,
    verbose=1, 
    mode='min',
    restore_best_weights=True
)
opt = Adam(lr=0.001)
autoencoder.compile(optimizer=opt, loss=decorr(input_layer, decoder, sample_weights,0.5))
# Train model
autoencoder.fit(x=normal_train_data, y=normal_train_data, epochs=20, batch_size=10000, validation_split=0.1)
history = autoencoder.fit([normal_train_data,normal_p_train], [normal_train_data,normal_p_train],
                    epochs=nb_epoch,
                    batch_size=batch_size,
                    shuffle=True,
                    validation_data=([test_data,p_test], [test_data,p_test]),
                    verbose=1,
                    ).history
# load and split dataset
input_dir = "/work/creissel/DATA/binaryclassifier"
allX = { feat : np.load(input_dir+'/%s.npy' % feat) for feat in ["jets","leps","met"] }
X = list(allX.values())
y = np.load(input_dir+'/target.npy')

from sklearn.model_selection import train_test_split
split = train_test_split(*X,y , test_size=0.1, random_state=42)
train = [ split[ix] for ix in range(0,len(split),2) ]
test = [ split[ix] for ix in range(1,len(split),2) ]
X_train, y_train = train[0:3], train[-1]
X_test, y_test = test[0:3], test[-1]

X_train.append(np.ones(len(y_train)))
X_test.append(np.ones(len(y_train)))

# load and split dataset

allX = { feat : np.genfromtxt('%s.csv' % feat, delimiter=',') for feat in ["Input_Background_1",
                                                                           "Input_Signal_1"]}
type(allx)
allx
X = list(allX.values())
y = np.load(input_dir+'/target.npy')

from sklearn.model_selection import train_test_split
split = train_test_split(*X,y , test_size=0.1, random_state=42)
train = [ split[ix] for ix in range(0,len(split),2) ]
test = [ split[ix] for ix in range(1,len(split),2) ]
X_train, y_train = train[0:3], train[-1]
X_test, y_test = test[0:3], test[-1]

X_train.append(np.ones(len(y_train)))
X_test.append(np.ones(len(y_train)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15874787672894927, 'fraction_numerical': 0.026983741810240235, 'mean_word_length': 4.65632720285479, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '46336', 'license': 'MIT', 'n_tokens_mistral': 6941, 'n_tokens_neox': 6445, 'n_words': 1525, 'path': 'Python/Anomaly_Detection/Autoenconder_keras/.ipynb_checkpoints/Understanding_Disco-checkpoint.ipynb', 'repo_name': 'mmaltafe/Learning', 'size': '20605'}