<jupyter_start><jupyter_text>K-Means<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
from sklearn.cluster import KMeans
from statsmodels.tsa.ar_model import AR
from sklearn.metrics import mean_squared_error
from math import sqrt
from pandas import read_csv
import pandas as pd
from pandas import DataFrame
from pandas import concat
import numpy as np 
from matplotlib import pyplot


class KMEANS:
    def __init__(self,path, window_width, n_clusters, train_rate):
        self.df = read_csv(path, header=0, index_col=0, parse_dates=True,squeeze=True)
        self.df = self.df.reset_index(drop=True)
        self.df.rename(columns={'anomaly':'is_anomaly'}, inplace=True)
        self.n_clusters = n_clusters
        self.window_width = window_width
        series = pd.DataFrame(self.df.iloc[:,0].values)  
        self.values = DataFrame(series.values)
        self.dataframe = concat([self.values.shift(1), self.values], axis=1)
        self.dataframe.columns = ['t', 't+1']

        self.train_size = int(len(self.values) * train_rate)

        self.train, self.test = self.dataframe.values[1:self.train_size], self.dataframe.values[self.train_size:]
        self.train_X, self.train_y = self.train[:,0], self.train[:,1]
        self.test_X, self.test_y = self.test[:,0], self.test[:,1]     
        self.create_persistence()

        X = series.values
        self.train, self.test = X[1:self.train_size], X[self.train_size:]        

    def model_persistence(self, x):
        return x
        
    def create_persistence(self):
        predictions = list()
        for x in self.test_X:
            yhat = self.model_persistence(x)
            predictions.append(yhat)
        rmse = sqrt(mean_squared_error(self.test_y, predictions))
#         print('Train shape', self.train_X.shape, ' - Test shape:' , self.test_X.shape)
#         print('Persistent Model RMSE: %.3f' % rmse)   

    def fit(self):
        self.create_sequence_vectors()
        self.cluster()
        self.computeErrorValues()

    def create_sequence_vectors(self):
        self.vectors = np.array([])
        for i,_ in enumerate(self.test[:-self.window_width+1]):

            if i==0:
                self.vectors =self.test[i:i+self.window_width].reshape(1,-1)
                continue
            # np.append(self.test[i:i+self.window_width], axis=0)
            self.vectors = np.append(self.vectors,self.test[i:i+self.window_width].reshape(1,-1),axis=0)


    def cluster(self):
        self.kmeans = KMeans(n_clusters=self.n_clusters).fit(np.array(self.vectors))
        self.vector_dist = []
        for i,_ in enumerate(self.vectors):
            min_dist = float("inf")
            for cluster in self.kmeans.cluster_centers_:
                dist = np.linalg.norm(self.vectors[i]-cluster)
                if min_dist> dist:
                    min_dist = dist
            self.vector_dist.append(min_dist)

    def computeErrorValues(self):
        self.errors = np.zeros_like(self.test)
        for i,_ in enumerate(self.test):
            start_index = max(0,i-self.window_width+1)
            end_index = min(len(self.vectors), i+self.window_width)
            self.errors[i] = np.mean(np.array(self.vector_dist[start_index:end_index]))


    def plot(self):
        # plot predicted error
        pyplot.figure(figsize=(50,5))
        pyplot.plot(self.test)
        # pyplot.plot(self.predictions, color='red')
        pyplot.plot(self.errors, color = 'yellow')
        pyplot.show()

    def get_roc_auc(self,verbose = True, plot=True):
        # get the predicted errors of the anomaly points
        indices = self.df[self.df['is_anomaly']==1].index >self.train_size
        true_anomaly_predicted_errors = self.errors[self.df[self.df['is_anomaly']==1].index[indices] - self.train_size ]
        if len(true_anomaly_predicted_errors) == 0:
            return np.nan
        # sort them 
        true_anomaly_predicted_errors = np.sort(true_anomaly_predicted_errors,axis=0).reshape(-1)
        true_anomaly_predicted_errors_extended = np.r_[np.linspace(0,true_anomaly_predicted_errors[0],40)[:-1],true_anomaly_predicted_errors]
                # now iterate thru the predicted errors from small to big
        # for each value look how much other points have equal or bigger error
        FPR = [] # fp/n  https://en.wikipedia.org/wiki/Sensitivity_and_specificity
        TPR = [] # tp/p
        p = len(true_anomaly_predicted_errors)
        Thresholds = []
        for predictederror in true_anomaly_predicted_errors_extended:
            threshold = predictederror
            tp = len(true_anomaly_predicted_errors[true_anomaly_predicted_errors>= threshold])
            fp = len(self.errors[self.errors>=threshold])-len(true_anomaly_predicted_errors[true_anomaly_predicted_errors>=threshold])

            fpr =fp/len(self.errors)
            FPR.append(fpr)
            TPR.append(tp/p)
            if verbose:
                print("Threshold: {0:25}  - FP: {1:4} - TP: {2:4} - FPR: {3:21} - TPR: {4:4}".format(threshold,fp, tp, fpr, tp/p))

        import matplotlib.pyplot as plt
        if plot:
            plt.figure()
            plt.axis([0, 1, 0, 1])
            plt.plot(FPR,TPR)
            plt.show() 

        # This is the AUC
        from sklearn.metrics import auc
        print('AUC: ' ,auc(FPR,TPR)        )
        return auc(FPR,TPR)

# kmeans = KMEANS('Univariate/YahooServiceNetworkTraffic/A2Benchmark/synthetic_45.csv',30,4,0.3)
# kmeans.fit()
# kmeans.plot()
# kmeans.get_roc_auc(verbose=False)<jupyter_output><empty_output><jupyter_text>Evaluation  Results for NYC_Taxi<jupyter_code># import os
# dfs = [pd.DataFrame(columns=['filename', 'auc']), pd.DataFrame(columns=['filename', 'auc']),pd.DataFrame(columns=['filename', 'auc']),pd.DataFrame(columns=['filename', 'auc'])]
# files = []
# import glob
# folders = ['A1Benchmark','A2Benchmark','A3Benchmark','A4Benchmark']
# for j,folder in enumerate(folders):
#     for i,file in enumerate(glob.glob('Univariate/YahooServiceNetworkTraffic/'+ folder + '/*.csv')):
#         print(file,end=' - ')
#         kmeans = KMEANS( file, 30,4,0.3)
#         kmeans.fit()
#         dfs[j].loc[i] = [os.path.basename(file),kmeans.get_roc_auc(plot=False,verbose=False)]




import os
import datetime

startTime = datetime.datetime.now()

import glob


cnn = KMEANS( 'drive/My Drive/MT/Experiments/Univariate/NYC_Taxi/nyc_taxi.csv',30,4,0.3)
cnn.fit()
# cnn.get_roc_auc(plot=False,verbose=False)

        
endTime = datetime.datetime.now()
diff = endTime - startTime
print('Time: ',diff)<jupyter_output>Time:  0:00:01.994550<jupyter_text>DBSCAN<jupyter_code>from sklearn.cluster import KMeans
from statsmodels.tsa.ar_model import AR
from sklearn.metrics import mean_squared_error
from math import sqrt
from pandas import read_csv
import pandas as pd
from pandas import DataFrame
from pandas import concat
import numpy as np 
from matplotlib import pyplot
from sklearn.cluster import DBSCAN
from sklearn import preprocessing

class DBSCAN_Compact:
    def __init__(self,path, window_width, eps, min_samples, train_rate):
        self.df = read_csv(path, header=0, index_col=0, parse_dates=True,squeeze=True)
        self.df = self.df.reset_index(drop=True)
        self.df.rename(columns={'anomaly':'is_anomaly'}, inplace=True)
        self.eps = eps
        self.min_samples = min_samples
        self.window_width = window_width
        series = pd.DataFrame(self.df.iloc[:,0].values)  
        self.values = DataFrame(series.values)
        self.dataframe = concat([self.values.shift(1), self.values], axis=1)
        self.dataframe.columns = ['t', 't+1']

        self.train_size = int(len(self.values) * train_rate)

        # train_labeled, test_labeled = self.dataframe.values[1:self.train_size], self.dataframe.values[self.train_size:]
        # self.train_X, self.train_y = train_labeled[:,0], train_labeled[:,1]
        # self.test_X, self.test_y = test_labeled[:,0], test_labeled[:,1]     
        # self.create_persistence()

        # X = series.values
        # self.train, self.test = X[1:self.train_size], X[self.train_size:]    

    def __build_sets(self):
        train_labeled, test_labeled = self.dataframe.values[1:self.train_size], self.dataframe.values[self.train_size:]
        self.train_X, self.train_y = train_labeled[:,0], train_labeled[:,1]
        self.test_X, self.test_y = test_labeled[:,0], test_labeled[:,1]   

        X = self.dataframe.iloc[:,1].values
        self.train, self.test = X[1:self.train_size], X[self.train_size:]    

    def standardize_dataframe(self):
        X = self.dataframe.values
        self.scalar = preprocessing.StandardScaler().fit(X)
        X = self.scalar.transform(X)
        self.dataframe = pd.DataFrame(X)

    def inverse_standardize_dataframe(self):
        X = self.dataframe.values
        X = self.scalar.inverse_transform(X)
        self.dataframe = pd.DataFrame(X)



    def model_persistence(self, x):
        return x
        
    def create_persistence(self):
        rmse = sqrt(mean_squared_error(self.dataframe['t'].iloc[self.train_size:], self.dataframe['t+1'].iloc[self.train_size::]))
#         print('Persistent Model RMSE: %.3f' % rmse)   

    def fit(self):
        self.create_persistence()
        self.standardize_dataframe()
        self.__build_sets()
                
        self.compute_anomalyScores()
        self.inverse_standardize_dataframe()
        
    def compute_anomalyScores(self):
        self.errors = np.zeros_like(self.test)
        # compute anomalies
        for i,_ in enumerate(self.test[:-self.window_width+1]):
            window = self.test[i:i+self.window_width]
            window2D = np.zeros((len(window),2))
            window2D[:,1] = window
            error = DBSCAN(self.eps, self.min_samples).fit_predict(window2D)
            error[error>0] = 0
            error*=-1
            self.errors[i:i+self.window_width] += error

        # normalize anomaly score
        self.errors[:-self.window_width+1] /= self.window_width
        for i,error in enumerate(self.test[-self.window_width+1:]):
            self.errors[-self.window_width + 1 + i] /=self.window_width-(i+1)


    def plot(self):
        # plot predicted error
        pyplot.figure(figsize=(50,5))
        pyplot.plot(self.test)
        # pyplot.plot(self.predictions, color='red')
        pyplot.plot(self.errors, color = 'red',  linewidth=0.5)
        pyplot.show()

    def get_roc_auc(self, plot=True, verbose=True):
        # get the predicted errors of the anomaly points
        indices = self.df[self.df['is_anomaly']==1].index >self.train_size
        true_anomaly_predicted_errors = self.errors[self.df[self.df['is_anomaly']==1].index[indices] - self.train_size ]
        if len(true_anomaly_predicted_errors) == 0:
            return np.nan
        # sort them 
        true_anomaly_predicted_errors = np.sort(true_anomaly_predicted_errors,axis=0).reshape(-1)
        true_anomaly_predicted_errors_extended = np.r_[np.linspace(0,true_anomaly_predicted_errors[0],40)[:-1],true_anomaly_predicted_errors]
        true_anomaly_predicted_errors_extended = np.r_[true_anomaly_predicted_errors_extended, true_anomaly_predicted_errors_extended[-1] + np.mean(true_anomaly_predicted_errors_extended)]
                # now iterate thru the predicted errors from small to big
        # for each value look how much other points have equal or bigger error
        FPR = [] # fp/n  https://en.wikipedia.org/wiki/Sensitivity_and_specificity
        TPR = [] # tp/p
        p = len(true_anomaly_predicted_errors)
        Thresholds = []
        for predictederror in true_anomaly_predicted_errors_extended:
            threshold = predictederror
            tp = len(true_anomaly_predicted_errors[true_anomaly_predicted_errors>= threshold])
            fp = len(self.errors[self.errors>=threshold])-len(true_anomaly_predicted_errors[true_anomaly_predicted_errors>=threshold])
            
            fpr =fp/len(self.errors)
            FPR.append(fpr)
            TPR.append(tp/p)
            if verbose:
                print("Threshold: {0:25}  - FP: {1:4} - TP: {2:4} - FPR: {3:21} - TPR: {4:4}".format(threshold,fp, tp, fpr, tp/p))

        import matplotlib.pyplot as plt
        if plot:
            plt.figure()
            plt.axis([0, 1, 0, 1])
            plt.plot(FPR,TPR)
            plt.show() 

        # This is the AUC
        from sklearn.metrics import auc
        print('AUC: ' ,auc(FPR,TPR)        )
        return auc(FPR,TPR)

# dbscan = DBSCAN_Compact('Univariate/YahooServiceNetworkTraffic/A2Benchmark/synthetic_45.csv',30,0.4,5,0.66)
# dbscan.fit()
# dbscan.plot()
# dbscan.get_roc_auc(verbose=False)<jupyter_output><empty_output><jupyter_text>Evaluation  Results for NYC_Taxi<jupyter_code># import os
# dfs = [pd.DataFrame(columns=['filename', 'auc']), pd.DataFrame(columns=['filename', 'auc']),pd.DataFrame(columns=['filename', 'auc']),pd.DataFrame(columns=['filename', 'auc'])]
# files = []
# import glob
# folders = ['A1Benchmark','A2Benchmark','A3Benchmark','A4Benchmark']
# for j,folder in enumerate(folders):
#     for i,file in enumerate(glob.glob('Univariate/YahooServiceNetworkTraffic/'+ folder + '/*.csv')):
#         print(file,end=' - ')
#         dbscan = DBSCAN_Compact( file, 30,0.4,5,0.3)
#         dbscan.fit()
#         dfs[j].loc[i] = [os.path.basename(file),dbscan.get_roc_auc(plot=False,verbose=False)]


import os
import datetime

startTime = datetime.datetime.now()

import glob


cnn = DBSCAN_Compact( 'drive/My Drive/MT/Experiments/Univariate/NYC_Taxi/nyc_taxi.csv', 70,0.9,16,0.3)
cnn.fit()
cnn.get_roc_auc(plot=False,verbose=False)
# cnn.compute_anomalyScores()
        
endTime = datetime.datetime.now()
diff = endTime - startTime
print('Time: ',diff)<jupyter_output>AUC:  0.4793804021535483
Time:  0:00:08.465314<jupyter_text>LOF<jupyter_code>import warnings
from sklearn.cluster import KMeans
from statsmodels.tsa.ar_model import AR
from sklearn.metrics import mean_squared_error
from math import sqrt
from pandas import read_csv
import pandas as pd
from pandas import DataFrame
from pandas import concat
import numpy as np 
from matplotlib import pyplot
from sklearn.neighbors import LocalOutlierFactor
from sklearn import preprocessing

class LOF_Compact:
    def __init__(self,path, window_width, n_neighbors, train_rate):
        self.df = read_csv(path, header=0, index_col=0, parse_dates=True,squeeze=True)
        self.df = self.df.reset_index(drop=True)
        self.df.rename(columns={'anomaly':'is_anomaly'}, inplace=True)
        self.n_neighbors = n_neighbors
        self.window_width = window_width
        series = pd.DataFrame(self.df.iloc[:,0].values)  
        self.values = DataFrame(series.values)
        self.dataframe = concat([self.values.shift(1), self.values], axis=1)
        self.dataframe.columns = ['t', 't+1']

        self.train_size = int(len(self.values) * train_rate)

        # train_labeled, test_labeled = self.dataframe.values[1:self.train_size], self.dataframe.values[self.train_size:]
        # self.train_X, self.train_y = train_labeled[:,0], train_labeled[:,1]
        # self.test_X, self.test_y = test_labeled[:,0], test_labeled[:,1]     
        # self.create_persistence()

        # X = series.values
        # self.train, self.test = X[1:self.train_size], X[self.train_size:]    

    def __build_sets(self):
        train_labeled, test_labeled = self.dataframe.values[1:self.train_size], self.dataframe.values[self.train_size:]
        self.train_X, self.train_y = train_labeled[:,0], train_labeled[:,1]
        self.test_X, self.test_y = test_labeled[:,0], test_labeled[:,1]   

        X = self.dataframe.iloc[:,1].values
        self.train, self.test = X[1:self.train_size], X[self.train_size:]    

    def standardize_dataframe(self):
        X = self.dataframe.values
        self.scalar = preprocessing.StandardScaler().fit(X)
        X = self.scalar.transform(X)
        self.dataframe = pd.DataFrame(X)

    def inverse_standardize_dataframe(self):
        X = self.dataframe.values
        X = self.scalar.inverse_transform(X)
        self.dataframe = pd.DataFrame(X)



    def model_persistence(self, x):
        return x
        
    def create_persistence(self):
        rmse = sqrt(mean_squared_error(self.dataframe['t'].iloc[self.train_size:], self.dataframe['t+1'].iloc[self.train_size::]))
#         print('Persistent Model RMSE: %.3f' % rmse)   

    def fit(self):
        self.create_persistence()
        self.standardize_dataframe()
        self.__build_sets()
                
        self.compute_anomalyScores()
        self.inverse_standardize_dataframe()
        
    def compute_anomalyScores(self):
        self.errors = np.zeros_like(self.test)
        # compute anomalies
        warnings.filterwarnings("ignore")

        for i,_ in enumerate(self.test[:-self.window_width+1]):
            window = self.test[i:i+self.window_width]
            window2D = np.zeros((len(window),2))
            window2D[:,1] = window
            clf = LocalOutlierFactor(self.n_neighbors,contamination ='auto')
            clf.fit_predict(window2D)
            error = clf.negative_outlier_factor_ 
            error=(error*-1) - 1
            self.errors[i:i+self.window_width] += error


        # normalize anomaly score
        self.errors[:-self.window_width+1] /= self.window_width
        for i,error in enumerate(self.test[-self.window_width+1:]):
            self.errors[-self.window_width + 1 + i] /=self.window_width-(i+1)

        scalar = preprocessing.MinMaxScaler((0,1)).fit(self.errors.reshape(-1,1))
        self.errors = scalar.transform(self.errors.reshape(-1,1))*10


    def plot(self):
        # plot predicted error
        pyplot.figure(figsize=(50,5))
        pyplot.plot(self.test)
        # pyplot.plot(self.predictions, color='red')
        pyplot.plot(self.errors, color = 'red',  linewidth=0.5)
        pyplot.show()

    def get_roc_auc(self, plot=True, verbose=True):
        # get the predicted errors of the anomaly points
        indices = self.df[self.df['is_anomaly']==1].index >self.train_size
        true_anomaly_predicted_errors = self.errors[self.df[self.df['is_anomaly']==1].index[indices] - self.train_size ]
        if len(true_anomaly_predicted_errors) == 0:
            return np.nan
        # sort them 
        true_anomaly_predicted_errors = np.sort(true_anomaly_predicted_errors,axis=0).reshape(-1)
        true_anomaly_predicted_errors_extended = np.r_[np.linspace(0,true_anomaly_predicted_errors[0],40)[:-1],true_anomaly_predicted_errors]
        true_anomaly_predicted_errors_extended = np.r_[true_anomaly_predicted_errors_extended, true_anomaly_predicted_errors_extended[-1] + np.mean(true_anomaly_predicted_errors_extended)]
                # now iterate thru the predicted errors from small to big
        # for each value look how much other points have equal or bigger error
        FPR = [] # fp/n  https://en.wikipedia.org/wiki/Sensitivity_and_specificity
        TPR = [] # tp/p
        p = len(true_anomaly_predicted_errors)
        Thresholds = []
        for predictederror in true_anomaly_predicted_errors_extended:
            threshold = predictederror
            tp = len(true_anomaly_predicted_errors[true_anomaly_predicted_errors>= threshold])
            fp = len(self.errors[self.errors>=threshold])-len(true_anomaly_predicted_errors[true_anomaly_predicted_errors>=threshold])
            
            fpr =fp/len(self.errors)
            FPR.append(fpr)
            TPR.append(tp/p)
            if verbose:
                print("Threshold: {0:25}  - FP: {1:4} - TP: {2:4} - FPR: {3:21} - TPR: {4:4}".format(threshold,fp, tp, fpr, tp/p))

        import matplotlib.pyplot as plt
        if plot:
            plt.figure()
            plt.axis([0, 1, 0, 1])
            plt.plot(FPR,TPR)
            plt.show() 

        # This is the AUC
        from sklearn.metrics import auc
        print('AUC: ' ,auc(FPR,TPR)        )
        return auc(FPR,TPR)
    
lof = LOF_Compact('Univariate/YahooServiceNetworkTraffic/A2Benchmark/synthetic_45.csv',30,10,0.66)
lof.fit()
lof.plot()
lof.get_roc_auc(verbose=False)<jupyter_output><empty_output><jupyter_text>Evaluation  Results for NYC_Taxi<jupyter_code># import os
# dfs = [pd.DataFrame(columns=['filename', 'auc']), pd.DataFrame(columns=['filename', 'auc']),pd.DataFrame(columns=['filename', 'auc']),pd.DataFrame(columns=['filename', 'auc'])]
# files = []
# import glob
# folders = ['A1Benchmark','A2Benchmark','A3Benchmark','A4Benchmark']
# for j,folder in enumerate(folders):
#     for i,file in enumerate(glob.glob('Univariate/YahooServiceNetworkTraffic/'+ folder + '/*.csv')):
#         print(file,end=' - ')
#         lof = LOF_Compact( file, 30,10,0.3)
#         lof.fit()
#         dfs[j].loc[i] = [os.path.basename(file),lof.get_roc_auc(plot=False,verbose=False)]
import os
import datetime

startTime = datetime.datetime.now()

import glob


cnn = LOF_Compact( 'drive/My Drive/MT/Experiments/Univariate/NYC_Taxi/nyc_taxi.csv',  20,5,0.3)
cnn.fit()
cnn.get_roc_auc(plot=False,verbose=False)

        
endTime = datetime.datetime.now()
diff = endTime - startTime
print('Time: ',diff)<jupyter_output>AUC:  0.5294685081872444
Time:  0:00:05.656312<jupyter_text>Isolation Forest<jupyter_code>import warnings
from sklearn.cluster import KMeans
from statsmodels.tsa.ar_model import AR
from sklearn.metrics import mean_squared_error
from math import sqrt
from pandas import read_csv
import pandas as pd
from pandas import DataFrame
from pandas import concat
import numpy as np 
from matplotlib import pyplot
from sklearn.ensemble import IsolationForest 
from sklearn import preprocessing
import sys

class IForest:
    def __init__(self,path, window_width, n_estimators, contamination, train_rate):
        self.df = read_csv(path, header=0, index_col=0, parse_dates=True,squeeze=True)
        self.df = self.df.reset_index(drop=True)
        self.df.rename(columns={'anomaly':'is_anomaly'}, inplace=True)
        self.n_estimators = n_estimators
        self.contamination = contamination
        self.window_width = window_width
        series = pd.DataFrame(self.df.iloc[:,0].values)  
        self.values = DataFrame(series.values)
        self.dataframe = concat([self.values.shift(1), self.values], axis=1)
        self.dataframe.columns = ['t', 't+1']

        self.train_size = int(len(self.values) * train_rate)

        # train_labeled, test_labeled = self.dataframe.values[1:self.train_size], self.dataframe.values[self.train_size:]
        # self.train_X, self.train_y = train_labeled[:,0], train_labeled[:,1]
        # self.test_X, self.test_y = test_labeled[:,0], test_labeled[:,1]     
        # self.create_persistence()

        # X = series.values
        # self.train, self.test = X[1:self.train_size], X[self.train_size:]    

    def __build_sets(self):
        train_labeled, test_labeled = self.dataframe.values[1:self.train_size], self.dataframe.values[self.train_size:]
        self.train_X, self.train_y = train_labeled[:,0], train_labeled[:,1]
        self.test_X, self.test_y = test_labeled[:,0], test_labeled[:,1]   

        X = self.dataframe.iloc[:,1].values
        self.train, self.test = X[1:self.train_size], X[self.train_size:]    

    def standardize_dataframe(self):
        X = self.dataframe.values
        self.scalar = preprocessing.StandardScaler().fit(X)
        X = self.scalar.transform(X)
        self.dataframe = pd.DataFrame(X)

    def inverse_standardize_dataframe(self):
        X = self.dataframe.values
        X = self.scalar.inverse_transform(X)
        self.dataframe = pd.DataFrame(X)



    def model_persistence(self, x):
        return x
        
    def create_persistence(self):
        rmse = sqrt(mean_squared_error(self.dataframe['t'].iloc[self.train_size:], self.dataframe['t+1'].iloc[self.train_size::]))
#         print('Persistent Model RMSE: %.3f' % rmse)   

    def fit(self):
        self.create_persistence()
        self.standardize_dataframe()
        self.__build_sets()
                
        self.compute_anomalyScores()
        self.inverse_standardize_dataframe()
    
    def getWindowedVectors(self, X):
        vectors = []
        for i,_ in enumerate(X[:-self.window_width+1]):
            vectors.append(X[i:i+self.window_width])
        return vectors

    def compute_anomalyScores(self):
        self.errors = np.zeros_like(self.test)
        # compute anomalies
        warnings.filterwarnings("ignore")

        # history = self.getWindowedVectors(self.train)

        for i,_ in enumerate(self.test[:-self.window_width+1]):
            sys.stdout.write('\r'+str(i)+':'+str(len(self.test) - self.window_width))

            window = self.test[i:i+self.window_width]
            window2D = np.zeros((len(window),2))
            window2D[:,1] = window
            clf=IsolationForest(n_estimators=self.n_estimators,   # number of isolation trees
                            max_samples='auto', # number of samples to draw to create an isolation tree
                            contamination=self.contamination, # porportion of outliers
                            max_features=1.0,   #
                            bootstrap=False, 
                            n_jobs=-1, 
                            random_state=42, 
                            verbose=0, behaviour='new')
            clf.fit(window2D)
            error = clf.decision_function(window2D) 
            error[error>0] = 0
            self.errors[i:i+self.window_width] += error*-1


        # normalize anomaly score
        self.errors[:-self.window_width+1] /= self.window_width
        for i,error in enumerate(self.test[-self.window_width+1:]):
            self.errors[-self.window_width + 1 + i] /=self.window_width-(i+1)

        self.errors_original = self.errors
        scalar = preprocessing.MinMaxScaler((0,1)).fit(self.errors.reshape(-1,1))
        self.errors = scalar.transform(self.errors.reshape(-1,1))*10


    def plot(self):
        # plot predicted error
        pyplot.figure(figsize=(50,5))
        pyplot.plot(self.test)
        # pyplot.plot(self.predictions, color='red')
        pyplot.plot(self.errors, color = 'red',  linewidth=0.5)
        pyplot.show()

    def get_roc_auc(self, plot=True, verbose=True):
        # get the predicted errors of the anomaly points
        indices = self.df[self.df['is_anomaly']==1].index >self.train_size
        true_anomaly_predicted_errors = self.errors[self.df[self.df['is_anomaly']==1].index[indices] - self.train_size ]
        if len(true_anomaly_predicted_errors) == 0:
            return np.nan

        # sort them 
        true_anomaly_predicted_errors = np.sort(true_anomaly_predicted_errors,axis=0).reshape(-1)
        true_anomaly_predicted_errors_extended = np.r_[np.linspace(0,true_anomaly_predicted_errors[0],40)[:-1],true_anomaly_predicted_errors]
        true_anomaly_predicted_errors_extended = np.r_[true_anomaly_predicted_errors_extended, true_anomaly_predicted_errors_extended[-1] + np.mean(true_anomaly_predicted_errors_extended)]
                # now iterate thru the predicted errors from small to big
        # for each value look how much other points have equal or bigger error
        FPR = [] # fp/n  https://en.wikipedia.org/wiki/Sensitivity_and_specificity
        TPR = [] # tp/p
        p = len(true_anomaly_predicted_errors)
        Thresholds = []
        for predictederror in true_anomaly_predicted_errors_extended:
            threshold = predictederror
            tp = len(true_anomaly_predicted_errors[true_anomaly_predicted_errors>= threshold])
            fp = len(self.errors[self.errors>=threshold])-len(true_anomaly_predicted_errors[true_anomaly_predicted_errors>=threshold])
            
            fpr =fp/len(self.errors)
            FPR.append(fpr)
            TPR.append(tp/p)
            if verbose:
                print("Threshold: {0:25}  - FP: {1:4} - TP: {2:4} - FPR: {3:21} - TPR: {4:4}".format(threshold,fp, tp, fpr, tp/p))

        import matplotlib.pyplot as plt
        if plot:
            plt.figure()
            plt.axis([0, 1, 0, 1])
            plt.plot(FPR,TPR)
            plt.show() 

        # This is the AUC
        from sklearn.metrics import auc
        print('AUC: ' ,auc(FPR,TPR)        )
        return auc(FPR,TPR)

# iforest = IForest('Univariate/YahooServiceNetworkTraffic/A2Benchmark/synthetic_45.csv',30,10,'auto',0.66)
# iforest.fit()
# iforest.plot()
# iforest.get_roc_auc(verbose=False)<jupyter_output><empty_output><jupyter_text>Evaluation  Results for NYC_Taxi<jupyter_code># import os
# dfs = [pd.DataFrame(columns=['filename', 'auc']), pd.DataFrame(columns=['filename', 'auc']),pd.DataFrame(columns=['filename', 'auc']),pd.DataFrame(columns=['filename', 'auc'])]
# files = []
# import glob
# folders = ['A1Benchmark','A2Benchmark','A3Benchmark','A4Benchmark']
# for j,folder in enumerate(folders):
#     for i,file in enumerate(glob.glob('Univariate/YahooServiceNetworkTraffic/'+ folder + '/*.csv')):
#         print(file,end=' - ')
#         iforest = IForest( file, 30,10, 'auto',0.3)
#         iforest.fit()
#         dfs[j].loc[i] = [os.path.basename(file),iforest.get_roc_auc(plot=False,verbose=False)]

import os
import datetime

startTime = datetime.datetime.now()

import glob


cnn = IForest( 'drive/My Drive/MT/Experiments/Univariate/NYC_Taxi/nyc_taxi.csv', 20,10, 'auto',0.3)
cnn.fit()
cnn.get_roc_auc(plot=False,verbose=False)

        
endTime = datetime.datetime.now()
diff = endTime - startTime
print('Time: ',diff)             
startTime = datetime.datetime.now()
cnn = IForest( 'drive/My Drive/MT/Experiments/Univariate/NYC_Taxi/nyc_taxi.csv', 20,10, 'auto',0.3)

cnn.fit()
endTime = datetime.datetime.now()
diff = endTime - startTime
print('Time: ',diff)<jupyter_output>7204:7204Time:  0:13:12.374665<jupyter_text>OCSVN<jupyter_code>import warnings
from sklearn.cluster import KMeans
from statsmodels.tsa.ar_model import AR
from sklearn.metrics import mean_squared_error
from math import sqrt
from pandas import read_csv
import pandas as pd
from pandas import DataFrame
from pandas import concat
import numpy as np 
from matplotlib import pyplot
from sklearn.svm import OneClassSVM
from sklearn import preprocessing
import sys

class OneClassSVM_AnomalyDetection:
    def __init__(self,path, window_width, nu, train_rate):
        self.df = read_csv(path, header=0, index_col=0, parse_dates=True,squeeze=True)
        self.df = self.df.reset_index(drop=True)
        self.df.rename(columns={'anomaly':'is_anomaly'}, inplace=True)
        self.nu = nu
        self.window_width = window_width
        series = pd.DataFrame(self.df.iloc[:,0].values)  
        self.values = DataFrame(series.values)
        self.dataframe = concat([self.values.shift(1), self.values], axis=1)
        self.dataframe.columns = ['t', 't+1']

        self.train_size = int(len(self.values) * train_rate)

        # train_labeled, test_labeled = self.dataframe.values[1:self.train_size], self.dataframe.values[self.train_size:]
        # self.train_X, self.train_y = train_labeled[:,0], train_labeled[:,1]
        # self.test_X, self.test_y = test_labeled[:,0], test_labeled[:,1]     
        # self.create_persistence()

        # X = series.values
        # self.train, self.test = X[1:self.train_size], X[self.train_size:]    

    def __build_sets(self):
        train_labeled, test_labeled = self.dataframe.values[1:self.train_size], self.dataframe.values[self.train_size:]
        self.train_X, self.train_y = train_labeled[:,0], train_labeled[:,1]
        self.test_X, self.test_y = test_labeled[:,0], test_labeled[:,1]   

        X = self.dataframe.iloc[:,1].values
        self.train, self.test = X[1:self.train_size], X[self.train_size:]    

    def standardize_dataframe(self):
        X = self.dataframe.values
        self.scalar = preprocessing.StandardScaler().fit(X)
        X = self.scalar.transform(X)
        self.dataframe = pd.DataFrame(X)

    def inverse_standardize_dataframe(self):
        X = self.dataframe.values
        X = self.scalar.inverse_transform(X)
        self.dataframe = pd.DataFrame(X)



    def model_persistence(self, x):
        return x
        
    def create_persistence(self):
        rmse = sqrt(mean_squared_error(self.dataframe['t'].iloc[self.train_size:], self.dataframe['t+1'].iloc[self.train_size::]))
#         print('Persistent Model RMSE: %.3f' % rmse)   

    def fit(self):
        self.create_persistence()
        self.standardize_dataframe()
        self.__build_sets()
                
        self.compute_anomalyScores()
        self.inverse_standardize_dataframe()
    
    def getWindowedVectors(self, X):
        vectors = []
        for i,_ in enumerate(X[:-self.window_width+1]):
            vectors.append(X[i:i+self.window_width])
        return vectors

    def compute_anomalyScores(self):
        self.errors = np.zeros_like(self.test)
        # compute anomalies
        warnings.filterwarnings("ignore")

        # history = self.getWindowedVectors(self.train)

        for i,_ in enumerate(self.test[:-self.window_width+1]):
            sys.stdout.write('\r'+str(i)+':'+str(len(self.test) - self.window_width))

            window = self.test[i:i+self.window_width]
            window2D = np.zeros((len(window),2))
            window2D[:,1] = window
            clf=OneClassSVM(nu=self.nu)
            clf.fit(window2D)
            error = clf.decision_function(window2D) 
            error[error>0] = 0
            self.errors[i:i+self.window_width] += error*-10


        # normalize anomaly score
        self.errors[:-self.window_width+1] /= self.window_width
        for i,error in enumerate(self.test[-self.window_width+1:]):
            self.errors[-self.window_width + 1 + i] /=self.window_width-(i+1)

        # self.errors_original = self.errors
        # scalar = preprocessing.MinMaxScaler((0,1)).fit(self.errors.reshape(-1,1))
        # self.errors = scalar.transform(self.errors.reshape(-1,1))*10


    def plot(self):
        # plot predicted error
        pyplot.figure(figsize=(50,5))
        pyplot.plot(self.test)
        # pyplot.plot(self.predictions, color='red')
        pyplot.plot(self.errors, color = 'red',  linewidth=0.5)
        pyplot.show()

    def get_roc_auc(self, plot=True, verbose=True):
        # get the predicted errors of the anomaly points
        indices = self.df[self.df['is_anomaly']==1].index >self.train_size
        true_anomaly_predicted_errors = self.errors[self.df[self.df['is_anomaly']==1].index[indices] - self.train_size ]
        if len(true_anomaly_predicted_errors) == 0:
            return np.nan
        # sort them 
        true_anomaly_predicted_errors = np.sort(true_anomaly_predicted_errors,axis=0).reshape(-1)
        true_anomaly_predicted_errors_extended = np.r_[np.linspace(0,true_anomaly_predicted_errors[0],40)[:-1],true_anomaly_predicted_errors]
        true_anomaly_predicted_errors_extended = np.r_[true_anomaly_predicted_errors_extended, true_anomaly_predicted_errors_extended[-1] + np.mean(true_anomaly_predicted_errors_extended)]
                # now iterate thru the predicted errors from small to big
        # for each value look how much other points have equal or bigger error
        FPR = [] # fp/n  https://en.wikipedia.org/wiki/Sensitivity_and_specificity
        TPR = [] # tp/p
        p = len(true_anomaly_predicted_errors)
        Thresholds = []
        for predictederror in true_anomaly_predicted_errors_extended:
            threshold = predictederror
            tp = len(true_anomaly_predicted_errors[true_anomaly_predicted_errors>= threshold])
            fp = len(self.errors[self.errors>=threshold])-len(true_anomaly_predicted_errors[true_anomaly_predicted_errors>=threshold])
            
            fpr =fp/len(self.errors)
            FPR.append(fpr)
            TPR.append(tp/p)
            if verbose:
                print("Threshold: {0:25}  - FP: {1:4} - TP: {2:4} - FPR: {3:21} - TPR: {4:4}".format(threshold,fp, tp, fpr, tp/p))

        import matplotlib.pyplot as plt
        if plot:
            plt.figure()
            plt.axis([0, 1, 0, 1])
            plt.plot(FPR,TPR)
            plt.show() 

        # This is the AUC
        from sklearn.metrics import auc
        print('AUC: ' ,auc(FPR,TPR)        )
        return auc(FPR,TPR)
    
iforest = OneClassSVM_AnomalyDetection('Univariate/YahooServiceNetworkTraffic/A1Benchmark/real_1.csv',30,0.7,0.3)
iforest.fit()
iforest.plot()
iforest.get_roc_auc(verbose=False)<jupyter_output>964:964<jupyter_text>Evaluation<jupyter_code>import os
import datetime
startTime = datetime.datetime.now()

dfs = [pd.DataFrame(columns=['filename', 'auc']), pd.DataFrame(columns=['filename', 'auc']),pd.DataFrame(columns=['filename', 'auc']),pd.DataFrame(columns=['filename', 'auc'])]
files = []
import glob
folders = ['A1Benchmark','A2Benchmark','A3Benchmark','A4Benchmark']
for j,folder in enumerate(folders):
    for i,file in enumerate(glob.glob('Univariate/YahooServiceNetworkTraffic/'+ folder + '/*.csv')):
        print(file,end=' - ')
        ocsvm = OneClassSVM_AnomalyDetection( file, 30,0.7,0.3)
        ocsvm.fit()
        dfs[j].loc[i] = [os.path.basename(file),ocsvm.get_roc_auc(plot=False,verbose=False)]
        
endTime = datetime.datetime.now()
diff = endTime - startTime
print('Time: ',diff.seconds)<jupyter_output>964:964AUC:  0.9781187122736418raffic/A1Benchmark\real_1.csv - 
978:978AUC:  0.7187118437118437raffic/A1Benchmark\real_10.csv - 
978:978AUC:  0.9496397243107768raffic/A1Benchmark\real_11.csv - 
978:978AUC:  0.9980158730158729raffic/A1Benchmark\real_12.csv - 
978:978AUC:  0.9703483245149913raffic/A1Benchmark\real_13.csv - 
978:978AUC:  0.990451388888889NetworkTraffic/A1Benchmark\real_15.csv - 
993:993AUC:  0.997067448680352Traffic/A1Benchmark\real_16.csv - 
967:967AUC:  0.6112787702313991raffic/A1Benchmark\real_17.csv - 
967:967AUC:  0.5800706083006729etworkTraffic/A1Benchmark\real_19.csv - 
978:978AUC:  0.9841269841269841raffic/A1Benchmark\real_2.csv - 
966:966AUC:  0.9201198734331265raffic/A1Benchmark\real_20.csv - 
964:964AUC:  0.993209255533199Traffic/A1Benchmark\real_21.csv - 
964:964AUC:  0.7865047427421672raffic/A1Benchmark\real_22.csv - 
964:964AUC:  0.9790409121395036raffic/A1Benchmark\real_23.csv - 
993:993AUC:  0.7088628217660475raffic/A1Benchmark\real_24.csv - 
975:975AUC:  [...]<jupyter_text>Results for UD1 - UD4<jupyter_code>print(dfs[0].describe())
print(dfs[1].describe())
print(dfs[2].describe())
print(dfs[3].describe())<jupyter_output>auc
count  59.000000
mean    0.850292
std     0.210282
min     0.191341
25%     0.780710
50%     0.944333
75%     0.989205
max     0.999022
              auc
count  100.000000
mean     0.995276
std      0.003292
min      0.990452
25%      0.990955
50%      0.995980
75%      0.998995
max      0.998995
             auc
count  99.000000
mean    0.957272
std     0.052413
min     0.696003
25%     0.951990
50%     0.972045
75%     0.991348
max     0.999150
             auc
count  98.000000
mean    0.939444
std     0.063793
min     0.720982
25%     0.908677
50%     0.960945
75%     0.989046
max     0.999150<jupyter_text>XGBoosting<jupyter_code>import warnings
from sklearn.cluster import KMeans
from statsmodels.tsa.ar_model import AR
from sklearn.metrics import mean_squared_error
from math import sqrt
from pandas import read_csv
import pandas as pd
from pandas import DataFrame
from pandas import concat
import numpy as np 
from matplotlib import pyplot
from xgboost import XGBRegressor
from sklearn import preprocessing
import sys

class XGBRegressor_AnomalyDetection:
    def __init__(self,path, window_width, nu, train_rate):
        self.df = read_csv(path, header=0, index_col=0, parse_dates=True,squeeze=True)
        self.df = self.df.reset_index(drop=True)
        self.df.rename(columns={'anomaly':'is_anomaly'}, inplace=True)
        self.nu = nu
        self.window_width = window_width
        series = pd.DataFrame(self.df.iloc[:,0].values)  
        self.values = DataFrame(series.values)
        self.dataframe = concat([self.values.shift(1), self.values], axis=1)
        self.dataframe.columns = ['t', 't+1']

        self.train_size = int(len(self.values) * train_rate)

        # train_labeled, test_labeled = self.dataframe.values[1:self.train_size], self.dataframe.values[self.train_size:]
        # self.train_X, self.train_y = train_labeled[:,0], train_labeled[:,1]
        # self.test_X, self.test_y = test_labeled[:,0], test_labeled[:,1]     
        # self.create_persistence()

        # X = series.values
        # self.train, self.test = X[1:self.train_size], X[self.train_size:]    

    def __build_sets(self):
        train_labeled, test_labeled = self.dataframe.values[1:self.train_size], self.dataframe.values[self.train_size:]
        self.train_X, self.train_y = train_labeled[:,0], train_labeled[:,1]
        self.test_X, self.test_y = test_labeled[:,0], test_labeled[:,1]   

        X = self.dataframe.iloc[:,1].values
        self.train, self.test = X[1:self.train_size], X[self.train_size:]    

    def standardize_dataframe(self):
        X = self.dataframe.values
        self.scalar = preprocessing.StandardScaler().fit(X)
        X = self.scalar.transform(X)
        self.dataframe = pd.DataFrame(X)

    def inverse_standardize_dataframe(self):
        X = self.dataframe.values
        X = self.scalar.inverse_transform(X)
        self.dataframe = pd.DataFrame(X)



    def model_persistence(self, x):
        return x
        
    def create_persistence(self):
        rmse = sqrt(mean_squared_error(self.dataframe['t'].iloc[self.train_size:], self.dataframe['t+1'].iloc[self.train_size::]))
#         print('Persistent Model RMSE: %.3f' % rmse)   

    def fit(self):
        self.create_persistence()
        self.__build_sets()
                
        self.compute_anomalyScores()
    
    def getWindowedVectors(self, X):
        vectors = []
        for i,_ in enumerate(X[:-self.window_width+1]):
            vectors.append(X[i:i+self.window_width])
        return vectors

    def compute_anomalyScores(self):

        xgb = XGBRegressor()
        xgb.fit(self.train_X.reshape(-1,1),self.train_y.reshape(-1,1))

        self.predictions = xgb.predict(self.test_X.reshape(-1,1))
        rmse = sqrt(mean_squared_error(self.test, self.predictions))
        self.errors = np.absolute(self.test - np.array(self.predictions))
#         print('Prediction Test RMSE: %.3f' % rmse)
    

    def plot(self):
        # plot predicted error
        pyplot.figure(figsize=(50,5))
        pyplot.plot(self.test)
        pyplot.plot(self.predictions, color='blue')
        pyplot.plot(self.errors, color = 'red',  linewidth=0.5)
        pyplot.show()

    def get_roc_auc(self, plot=True, verbose=True):
        # get the predicted errors of the anomaly points
        indices = self.df[self.df['is_anomaly']==1].index >self.train_size
        true_anomaly_predicted_errors = self.errors[self.df[self.df['is_anomaly']==1].index[indices] - self.train_size ]
        if len(true_anomaly_predicted_errors) == 0:
            return np.nan
        # sort them 
        true_anomaly_predicted_errors = np.sort(true_anomaly_predicted_errors,axis=0).reshape(-1)
        true_anomaly_predicted_errors_extended = np.r_[np.linspace(0,true_anomaly_predicted_errors[0],40)[:-1],true_anomaly_predicted_errors]
        true_anomaly_predicted_errors_extended = np.r_[true_anomaly_predicted_errors_extended, true_anomaly_predicted_errors_extended[-1] + np.mean(true_anomaly_predicted_errors_extended)]
                # now iterate thru the predicted errors from small to big
        # for each value look how much other points have equal or bigger error
        FPR = [] # fp/n  https://en.wikipedia.org/wiki/Sensitivity_and_specificity
        TPR = [] # tp/p
        p = len(true_anomaly_predicted_errors)
        Thresholds = []
        for predictederror in true_anomaly_predicted_errors_extended:
            threshold = predictederror
            tp = len(true_anomaly_predicted_errors[true_anomaly_predicted_errors>= threshold])
            fp = len(self.errors[self.errors>=threshold])-len(true_anomaly_predicted_errors[true_anomaly_predicted_errors>=threshold])
            
            fpr =fp/len(self.errors)
            FPR.append(fpr)
            TPR.append(tp/p)
            if verbose:
                print("Threshold: {0:25}  - FP: {1:4} - TP: {2:4} - FPR: {3:21} - TPR: {4:4}".format(threshold,fp, tp, fpr, tp/p))

        import matplotlib.pyplot as plt
        if plot:
            plt.figure()
            plt.axis([0, 1, 0, 1])
            plt.plot(FPR,TPR)
            plt.show() 

        # This is the AUC
        from sklearn.metrics import auc
        print('AUC: ' ,auc(FPR,TPR)        )
        return auc(FPR,TPR)

iforest = XGBRegressor_AnomalyDetection('Univariate/YahooServiceNetworkTraffic/A1Benchmark/real_4.csv',30,0.7,0.66)
iforest.fit()
iforest.plot()
iforest.get_roc_auc(verbose=False)<jupyter_output>[16:29:29] WARNING: src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.<jupyter_text>Evaluation<jupyter_code>import os
import datetime
startTime = datetime.datetime.now()

dfs = [pd.DataFrame(columns=['filename', 'auc']), pd.DataFrame(columns=['filename', 'auc']),pd.DataFrame(columns=['filename', 'auc']),pd.DataFrame(columns=['filename', 'auc'])]
files = []
import glob
folders = ['A1Benchmark','A2Benchmark','A3Benchmark','A4Benchmark']
for j,folder in enumerate(folders):
    for i,file in enumerate(glob.glob('Univariate/YahooServiceNetworkTraffic/'+ folder + '/*.csv')):
        print(file,end=' - ')
        xgb = XGBRegressor_AnomalyDetection( file, 30,0.7,0.3)
        xgb.fit()
        dfs[j].loc[i] = [os.path.basename(file),xgb.get_roc_auc(plot=False,verbose=False)]
        
endTime = datetime.datetime.now()
diff = endTime - startTime
print('Time: ',diff.seconds)<jupyter_output>Univariate/YahooServiceNetworkTraffic/A1Benchmark\real_1.csv - [16:30:41] WARNING: src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.
AUC:  0.9979879275653925
Univariate/YahooServiceNetworkTraffic/A1Benchmark\real_10.csv - [16:30:41] WARNING: src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.
AUC:  0.9871031746031746
Univariate/YahooServiceNetworkTraffic/A1Benchmark\real_11.csv - [16:30:41] WARNING: src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.
AUC:  0.981124686716792
Univariate/YahooServiceNetworkTraffic/A1Benchmark\real_12.csv - [16:30:41] WARNING: src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.
AUC:  0.998015873015873
Univariate/YahooServiceNetworkTraffic/A1Benchmark\real_13.csv - [16:30:41] WARNING: src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.
AUC[...]<jupyter_text>Results for UD1 - UD4<jupyter_code>print(dfs[0].describe())
print(dfs[1].describe())
print(dfs[2].describe())
print(dfs[3].describe())<jupyter_output>auc
count  59.000000
mean    0.896743
std     0.157818
min     0.301238
25%     0.877046
50%     0.976931
75%     0.991539
max     0.999008
              auc
count  100.000000
mean     0.968308
std      0.083773
min      0.538135
25%      0.990955
50%      0.995917
75%      0.998492
max      0.998995
             auc
count  99.000000
mean    0.802310
std     0.195970
min     0.253295
25%     0.664045
50%     0.814433
75%     0.988544
max     0.998724
             auc
count  98.000000
mean    0.853750
std     0.162572
min     0.289683
25%     0.748914
50%     0.920123
75%     0.989266
max     0.999150<jupyter_text>XGBoosting Sliding<jupyter_code>import warnings
from sklearn.cluster import KMeans
from statsmodels.tsa.ar_model import AR
from sklearn.metrics import mean_squared_error
from math import sqrt
from pandas import read_csv
import pandas as pd
from pandas import DataFrame
from pandas import concat
import numpy as np 
from matplotlib import pyplot
from xgboost import XGBRegressor
from sklearn import preprocessing
import sys

def warn(*args, **kwargs):
    pass
    
class XGBRegressorSliding_AnomalyDetection:
    def __init__(self,path, window_width, nu, train_rate):
        self.df = read_csv(path, header=0, index_col=0, parse_dates=True,squeeze=True)
        self.df = self.df.reset_index(drop=True)
        self.df.rename(columns={'anomaly':'is_anomaly'}, inplace=True)
        self.nu = nu
        self.window_width = window_width
        series = pd.DataFrame(self.df.iloc[:,0].values)  
        self.values = DataFrame(series.values)
        self.dataframe = concat([self.values.shift(1), self.values], axis=1)
        self.dataframe.columns = ['t', 't+1']

        self.train_size = int(len(self.values) * train_rate)

        # train_labeled, test_labeled = self.dataframe.values[1:self.train_size], self.dataframe.values[self.train_size:]
        # self.train_X, self.train_y = train_labeled[:,0], train_labeled[:,1]
        # self.test_X, self.test_y = test_labeled[:,0], test_labeled[:,1]     
        # self.create_persistence()

        # X = series.values
        # self.train, self.test = X[1:self.train_size], X[self.train_size:]    

    def __build_sets(self):
        train_labeled, test_labeled = self.dataframe.values[1:self.train_size], self.dataframe.values[self.train_size:]
        self.train_X, self.train_y = train_labeled[:,0], train_labeled[:,1]
        self.test_X, self.test_y = test_labeled[:,0], test_labeled[:,1]   

        X = self.dataframe.iloc[:,1].values
        self.train, self.test = X[1:self.train_size], X[self.train_size:]    

    def standardize_dataframe(self):
        X = self.dataframe.values
        self.scalar = preprocessing.StandardScaler().fit(X)
        X = self.scalar.transform(X)
        self.dataframe = pd.DataFrame(X)

    def inverse_standardize_dataframe(self):
        X = self.dataframe.values
        X = self.scalar.inverse_transform(X)
        self.dataframe = pd.DataFrame(X)



    def model_persistence(self, x):
        return x
        
    def create_persistence(self):
        rmse = sqrt(mean_squared_error(self.dataframe['t'].iloc[self.train_size:], self.dataframe['t+1'].iloc[self.train_size::]))
#         print('Persistent Model RMSE: %.3f' % rmse)   

    def fit(self):
        self.create_persistence()
        self.__build_sets()
                
        self.compute_anomalyOptScores()
    
    def getWindowedVectors(self, X):
        vectors = np.zeros((len(X) - self.window_width+1,self.window_width))
        for i,_ in enumerate(X[:-self.window_width+1]):
            vectors[i] = X[i:i+self.window_width]
        return vectors

    def compute_anomalyScores(self):

        xgb = XGBRegressor()
        xgb.fit(self.train_X.reshape(-1,1),self.train_y.reshape(-1,1))

        self.predictions = xgb.predict(self.test_X.reshape(-1,1))
        rmse = sqrt(mean_squared_error(self.test, self.predictions))
        self.errors = np.absolute(self.test - np.array(self.predictions))
#         print('Prediction Test RMSE: %.3f' % rmse)

   
    def compute_anomalyOptScores(self):

        warnings.warn = warn 
        # compute anomalies
        self.X = self.getWindowedVectors(self.dataframe.values[1:,0])
        self.y = self.dataframe.values[-len(self.X):,1]
        self.train_X, self.train_y= self.X[:self.train_size], self.y[:self.train_size]
        self.test_X, self.test_y = self.X[self.train_size:], self.y[self.train_size:]

        # xgb = XGBRegressor()
        # xgb.fit(self.train_X, self.train_y.reshape(-1,1))
        self.history = (self.train_X.tolist(), self.train_y.tolist())
        self.predictions = []

        for i,test_elem in enumerate(self.test_X):
            # warnings.filterwarnings("ignore")
            xgb = XGBRegressor(objective ='reg:squarederror')
            xgb.fit(self.history[0],self.history[1])
            output = xgb.predict(test_elem.reshape(1,-1))
            self.predictions.append(output[0])
            self.history[0].append(self.test_X[i])
            self.history[1].append(self.test_y[i])
            # self.history = (self.history[0].append(self.test_X[i]),self.history[1].append(self.test_y[i]))
            sys.stdout.write('\r '+str(i)+':'+str(len(self.test_X)))
        print('')

        rmse = sqrt(mean_squared_error(self.test_y, self.predictions))
        self.errors = np.absolute(self.test_y - np.array(self.predictions))
#         print('Prediction Test RMSE: %.3f' % rmse)


    def plot(self):
        # plot predicted error
        pyplot.figure(figsize=(50,5))
        pyplot.plot(self.test_y)
        pyplot.plot(self.predictions, color='blue')
        pyplot.plot(self.errors, color = 'red',  linewidth=0.5)
        pyplot.show()

    def get_roc_auc(self, plot=True, verbose=True):
        # get the predicted errors of the anomaly points
        indices = self.df[self.df['is_anomaly']==1].index >self.train_size
        true_anomaly_predicted_errors = self.errors[self.df[self.df['is_anomaly']==1].index[indices] - self.train_size - self.window_width ]
        if len(true_anomaly_predicted_errors) == 0:
            return np.nan
        # sort them 
        true_anomaly_predicted_errors = np.sort(true_anomaly_predicted_errors,axis=0).reshape(-1)
        true_anomaly_predicted_errors_extended = np.r_[np.linspace(0,true_anomaly_predicted_errors[0],40)[:-1],true_anomaly_predicted_errors]
        true_anomaly_predicted_errors_extended = np.r_[true_anomaly_predicted_errors_extended, true_anomaly_predicted_errors_extended[-1] + np.mean(true_anomaly_predicted_errors_extended)]
                # now iterate thru the predicted errors from small to big
        # for each value look how much other points have equal or bigger error
        FPR = [] # fp/n  https://en.wikipedia.org/wiki/Sensitivity_and_specificity
        TPR = [] # tp/p
        p = len(true_anomaly_predicted_errors)
        Thresholds = []
        for predictederror in true_anomaly_predicted_errors_extended:
            threshold = predictederror
            tp = len(true_anomaly_predicted_errors[true_anomaly_predicted_errors>= threshold])
            fp = len(self.errors[self.errors>=threshold])-len(true_anomaly_predicted_errors[true_anomaly_predicted_errors>=threshold])
            
            fpr =fp/len(self.errors)
            FPR.append(fpr)
            TPR.append(tp/p)
            if verbose:
                print("Threshold: {0:25}  - FP: {1:4} - TP: {2:4} - FPR: {3:21} - TPR: {4:4}".format(threshold,fp, tp, fpr, tp/p))

        import matplotlib.pyplot as plt
        if plot:
            plt.figure()
            plt.axis([0, 1, 0, 1])
            plt.plot(FPR,TPR)
            plt.show() 

        # This is the AUC
        from sklearn.metrics import auc
        print('AUC: ' ,auc(FPR,TPR)        )
        return auc(FPR,TPR)

xgb = XGBRegressorSliding_AnomalyDetection('Univariate/YahooServiceNetworkTraffic/A1Benchmark/real_1.csv',30,0.7,0.66)
xgb.fit()
xgb.plot()
xgb.get_roc_auc(verbose=False)<jupyter_output>452:453<jupyter_text>Evaluation<jupyter_code>import os
import datetime
startTime = datetime.datetime.now()

dfs = [pd.DataFrame(columns=['filename', 'auc']), pd.DataFrame(columns=['filename', 'auc']),pd.DataFrame(columns=['filename', 'auc']),pd.DataFrame(columns=['filename', 'auc'])]
files = []
import glob
folders = ['A1Benchmark','A2Benchmark','A3Benchmark','A4Benchmark']
for j,folder in enumerate(folders):
    for i,file in enumerate(glob.glob('Univariate/YahooServiceNetworkTraffic/'+ folder + '/*.csv')):
        print(file,end=' - ')
        xgb = XGBRegressorSliding_AnomalyDetection( file, 30,0.7,0.3)
        xgb.fit()
        dfs[j].loc[i] = [os.path.basename(file),xgb.get_roc_auc(plot=False,verbose=False)]
        
endTime = datetime.datetime.now()
diff = endTime - startTime
print('Time: ',diff.seconds)<jupyter_output>963:964te/YahooServiceNetworkTraffic/A1Benchmark\real_1.csv - 
AUC:  0.9979253112033194
 977:978te/YahooServiceNetworkTraffic/A1Benchmark\real_10.csv - 
AUC:  0.7965628441088564
 977:978te/YahooServiceNetworkTraffic/A1Benchmark\real_11.csv - 
AUC:  0.9805725971370143
 977:978te/YahooServiceNetworkTraffic/A1Benchmark\real_12.csv - 
AUC:  0.9979550102249488
 977:978te/YahooServiceNetworkTraffic/A1Benchmark\real_13.csv - 
AUC:  0.9889229720518065
 977:978te/YahooServiceNetworkTraffic/A1Benchmark\real_14.csv - 
 977:978te/YahooServiceNetworkTraffic/A1Benchmark\real_15.csv - 
AUC:  0.9841513292433538
 992:993te/YahooServiceNetworkTraffic/A1Benchmark\real_16.csv - 
AUC:  0.9959718026183284
 966:967te/YahooServiceNetworkTraffic/A1Benchmark\real_17.csv - 
AUC:  0.652262549599333
 992:993te/YahooServiceNetworkTraffic/A1Benchmark\real_18.csv - 
 966:967te/YahooServiceNetworkTraffic/A1Benchmark\real_19.csv - 
AUC:  0.672503177546251
 977:978te/YahooServiceNetworkTraffic/A1Benchmark\real_2.csv - [...]<jupyter_text>Results for UD1 - UD4<jupyter_code>print(dfs[0].describe())
print(dfs[1].describe())
print(dfs[2].describe())
print(dfs[3].describe())<jupyter_output>auc
count  59.000000
mean    0.869522
std     0.210411
min     0.171356
25%     0.831157
50%     0.972686
75%     0.990025
max     0.998993
              auc
count  100.000000
mean     0.983792
std      0.029401
min      0.808405
25%      0.984542
50%      0.995596
75%      0.998964
max      0.998964
             auc
count  99.000000
mean    0.987954
std     0.022607
min     0.801483
25%     0.990382
50%     0.993642
75%     0.996128
max     0.999127
             auc
count  98.000000
mean    0.978157
std     0.037135
min     0.752182
25%     0.980212
50%     0.991638
75%     0.996510
max     0.999127
{'fraction_non_alphanumeric': 0.12631560725661264, 'fraction_numerical': 0.03998753635230577, 'mean_word_length': 3.0875256491898395, 'pattern_counts': {'":': 0, '<': 56, '<?xml version=': 0, '>': 88, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '219912', 'license': 'MIT', 'n_tokens_mistral': 21513, 'n_tokens_neox': 19715, 'n_words': 3215, 'path': 'Univariate/ML/NYCT/UnivariateML_NYCT.ipynb', 'repo_name': 'MbProg/Anomaly-detection-of-timeseries', 'size': '57768'}