<jupyter_start><jupyter_text>Unsupervised outliers detection (event detection)<jupyter_code>import drama as drm
import numpy as np
import matplotlib.pylab as plt
from matplotlib import gridspec

from drama.outlier_finder import grid_run_drama

from keras.datasets import mnist

%matplotlib inline

n_try = 5
# MNIST dataset
(x_train, y_train), (x_test, y_test) = mnist.load_data()

image_size = x_train.shape[1]
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255
print(x_train.shape, y_train.shape, x_test.shape, y_test.shape)
inlier_labels = [2,3,4,6,8]
outlier_labels = [5,0]
n_inliers = 500
n_outliers = 10

X = []
y = []
for i in inlier_labels:
    filt = y_train==i
    ns = np.sum(filt)
    X.extend(x_train[filt][:n_inliers])
    y.extend(n_inliers*[0])
    
for i in outlier_labels:
    filt = y_train==i
    ns = np.sum(filt)
    X.extend(x_train[filt][:n_outliers])
    y.extend(n_outliers*[1])
    
X = np.array(X)
y = np.array(y)
X.shape,y.shape
X = np.reshape(X, [-1, image_size*image_size])
lof_all = np.zeros((n_try,3))
ifr_all = np.zeros((n_try,3))
df = drm.sk_check(X.reshape(-1,784),X.reshape(-1,784),y,[1])
for i in range(n_try):
    for j,scr in enumerate(['AUC','MCC','RWS']):
        lof_all[i,j] = df[scr][0]
        ifr_all[i,j] = df[scr][1]
df<jupyter_output><empty_output><jupyter_text>Outlier detection<jupyter_code>metrics = ['cityblock', 'L2', 'L4', 'braycurtis', 'canberra', 'chebyshev',
                        'correlation', 'mahalanobis', 'wL2', 'wL4']
drt_list = ['DAE1D', 'DVAE1D']

result = []
for i in range(n_try):
#     auc,mcc,rws,conf = grid_run_drama(X,y)
    auc,mcc,rws,conf = grid_run_drama(X,y,
                                      drt_list=drt_list,
                                      metrics=metrics,
                                      n_split=2)
    
    arr = np.stack([auc,mcc,rws],axis=-1)
    result.append(arr)
result = np.array(result)

drts = np.unique(conf[:,1])
metrs = np.unique(conf[:,2])

res = result.reshape(n_try,len(drt_list),len(metrics),-1)

drm.plot_table(np.mean(res,axis=0),drts,metrs)

auc = np.sum((res[:, :, :, 0].T>lof_all[:, 0]) & (res[:, :, :, 0].T>ifr_all[:, 0]),axis=-1).T
mcc = np.sum((res[:, :, :, 1].T>lof_all[:, 1]) & (res[:, :, :, 1].T>ifr_all[:, 1]),axis=-1).T
rws = np.sum((res[:, :, :, 2].T>lof_all[:, 2]) & (res[:, :, :, 2].T>ifr_all[:, 2]),axis=-1).T

fig = plt.figure(figsize=(20,10))
plt.clf()
ax = fig.add_subplot(111)
ax.set_aspect('auto')
ax.imshow(auc, cmap=plt.cm.jet,interpolation='nearest')

width, height = auc.shape

for x in range(width):
    for y in range(height):
        ax.annotate('AUC: {:d}\n MCC: {:d}\n RWS: {:d}'.format(auc[x][y],mcc[x][y],rws[x][y]), xy=(y, x), 
                    horizontalalignment='center',
                    verticalalignment='center',fontsize=18);

plt.xticks(range(len(metrs)),metrs,fontsize=15)
plt.yticks(range(len(drts)), drts,fontsize=15)

plt.title('Number of successes (LOF and i-forest) out of 20 data set',fontsize=25)
plt.annotate('** Colors depend on AUC.', (0,0), (0, -30), xycoords='axes fraction',
             textcoords='offset points', va='top',fontsize=15)

# plt.savefig('AND_success.jpg',dpi=150,bbox_inches='tight')
inlier_labels = [1,2,3,4]
outlier_labels = [5]
n_inliers = 1000
n_outliers = 30

X = []
y = []
for i in inlier_labels:
    filt = y_train==i
    ns = np.sum(filt)
    X.extend(x_train[filt][:n_inliers])
    y.extend(n_inliers*[0])
    
for i in outlier_labels:
    filt = y_train==i
    ns = np.sum(filt)
    X.extend(x_train[filt][:n_outliers])
    y.extend(n_outliers*[1])
    
X = np.array(X)
y = np.array(y)
X = np.reshape(X, [-1, image_size*image_size, 1])
metrics = ['cityblock', 'L2', 'L4', 'braycurtis', 'canberra', 'chebyshev',
                        'correlation', 'mahalanobis', 'wL2', 'wL4']
drt_list = ['CAE1D', 'CVAE1D']

result = []
for i in range(n_try):
#     auc,mcc,rws,conf = grid_run_drama(X,y)
    auc,mcc,rws,conf = grid_run_drama(X,y,
                                      drt_list=drt_list,
                                      metrics=metrics,
                                      n_split=2)
    
    arr = np.stack([auc,mcc,rws],axis=-1)
    result.append(arr)
result = np.array(result)

drts = np.unique(conf[:,1])
metrs = np.unique(conf[:,2])

res = result.reshape(n_try,len(drt_list),len(metrics),-1)

drm.plot_table(np.mean(res,axis=0),drts,metrs)

auc = np.sum((res[:, :, :, 0].T>lof_all[:, 0]) & (res[:, :, :, 0].T>ifr_all[:, 0]),axis=-1).T
mcc = np.sum((res[:, :, :, 1].T>lof_all[:, 1]) & (res[:, :, :, 1].T>ifr_all[:, 1]),axis=-1).T
rws = np.sum((res[:, :, :, 2].T>lof_all[:, 2]) & (res[:, :, :, 2].T>ifr_all[:, 2]),axis=-1).T

fig = plt.figure(figsize=(20,10))
plt.clf()
ax = fig.add_subplot(111)
ax.set_aspect('auto')
ax.imshow(auc, cmap=plt.cm.jet,interpolation='nearest')

width, height = auc.shape

for x in range(width):
    for y in range(height):
        ax.annotate('AUC: {:d}\n MCC: {:d}\n RWS: {:d}'.format(auc[x][y],mcc[x][y],rws[x][y]), xy=(y, x), 
                    horizontalalignment='center',
                    verticalalignment='center',fontsize=18);

plt.xticks(range(len(metrs)),metrs,fontsize=15)
plt.yticks(range(len(drts)), drts,fontsize=15)

plt.title('Number of successes (LOF and i-forest) out of 20 data set',fontsize=25)
plt.annotate('** Colors depend on AUC.', (0,0), (0, -30), xycoords='axes fraction',
             textcoords='offset points', va='top',fontsize=15)

# plt.savefig('AND_success.jpg',dpi=150,bbox_inches='tight')
inlier_labels = [1,2,3,4]
outlier_labels = [5]
n_inliers = 1000
n_outliers = 30

X = []
y = []
for i in inlier_labels:
    filt = y_train==i
    ns = np.sum(filt)
    X.extend(x_train[filt][:n_inliers])
    y.extend(n_inliers*[0])
    
for i in outlier_labels:
    filt = y_train==i
    ns = np.sum(filt)
    X.extend(x_train[filt][:n_outliers])
    y.extend(n_outliers*[1])
    
X = np.array(X)
y = np.array(y)
X = np.reshape(X, [-1, image_size, image_size, 1])
metrics = ['cityblock', 'L2', 'L4', 'braycurtis', 'canberra', 'chebyshev',
                        'correlation', 'mahalanobis', 'wL2', 'wL4']
drt_list = ['CAE2D', 'CVAE2D']

result = []
for i in range(n_try):
#     auc,mcc,rws,conf = grid_run_drama(X,y)
    auc,mcc,rws,conf = grid_run_drama(X,y,
                                      drt_list=drt_list,
                                      metrics=metrics,
                                      n_split=2)
    
    arr = np.stack([auc,mcc,rws],axis=-1)
    result.append(arr)
result = np.array(result)

drts = np.unique(conf[:,1])
metrs = np.unique(conf[:,2])

res = result.reshape(n_try,len(drt_list),len(metrics),-1)

drm.plot_table(np.mean(res,axis=0),drts,metrs)

auc = np.sum((res[:, :, :, 0].T>lof_all[:, 0]) & (res[:, :, :, 0].T>ifr_all[:, 0]),axis=-1).T
mcc = np.sum((res[:, :, :, 1].T>lof_all[:, 1]) & (res[:, :, :, 1].T>ifr_all[:, 1]),axis=-1).T
rws = np.sum((res[:, :, :, 2].T>lof_all[:, 2]) & (res[:, :, :, 2].T>ifr_all[:, 2]),axis=-1).T

fig = plt.figure(figsize=(20,10))
plt.clf()
ax = fig.add_subplot(111)
ax.set_aspect('auto')
ax.imshow(auc, cmap=plt.cm.jet,interpolation='nearest')

width, height = auc.shape

for x in range(width):
    for y in range(height):
        ax.annotate('AUC: {:d}\n MCC: {:d}\n RWS: {:d}'.format(auc[x][y],mcc[x][y],rws[x][y]), xy=(y, x), 
                    horizontalalignment='center',
                    verticalalignment='center',fontsize=18);

plt.xticks(range(len(metrs)),metrs,fontsize=15)
plt.yticks(range(len(drts)), drts,fontsize=15)

plt.title('Number of successes (LOF and i-forest) out of 20 data set',fontsize=25)
plt.annotate('** Colors depend on AUC.', (0,0), (0, -30), xycoords='axes fraction',
             textcoords='offset points', va='top',fontsize=15)

# plt.savefig('AND_success.jpg',dpi=150,bbox_inches='tight')<jupyter_output>Unsupervised outlier detection mode.
{'fraction_non_alphanumeric': 0.2046032553407935, 'fraction_numerical': 0.029501525940996948, 'mean_word_length': 3.4891552511415527, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '3164', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3507, 'n_tokens_neox': 3338, 'n_words': 539, 'path': 'notebooks/trial/MNIST.ipynb', 'repo_name': 'vafaei-ar/pymce', 'size': '7864'}