<jupyter_start><jupyter_text>数据挖掘第四次课程作业       吴嘉豪   _3220190894_  数据异常点检测  1. python库引入<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.utils import shuffle

from pyod.models.pca import PCA
from pyod.models.knn import KNN
from pyod.models.lof import LOF
from pyod.models.iforest import IForest

from time import *
import sys
import os

from sklearn.metrics import roc_auc_score
from sklearn.utils import column_or_1d
from sklearn.utils import check_consistent_length
from pyod.utils.utility import precision_n_scores

import warnings
warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>2. 处理数据集  __1. 数据集分析 - skin_benchmarks:__   该数据集中包含多条skin_point的数据记录，数据中给出了skin_point所对应的RGB数值，并且由GT于origin.label提供了每一条记录点是否为异常点的记录，可以由此作为test集合上进行验证的label。<jupyter_code>benchmark_dir1 = "F:\\course\\web\\homework4\\homework4\\skin\\benchmarks\\"
benchmark_dir = []
for files in os.listdir(benchmark_dir1):
    file_path = os.path.join(benchmark_dir1, files)
    benchmark_dir.append(file_path)
np.random.shuffle(benchmark_dir)
train_rate = 0.7
total = len(benchmark_dir)
print("Benchmark number : " , total)
print("Example : ", benchmark_dir[0])
data = pd.read_csv(benchmark_dir[0])
print("Benchmark size:", data.shape)
data.head()<jupyter_output>Benchmark number :  1500
Example :  F:\course\web\homework4\homework4\skin\benchmarks\skin_benchmark_1389.csv
Benchmark size: (6000, 10)<jupyter_text>由上面的数据读取可以看出，在实验中使用了该数据总计包括1500个benchmark，每个benchmark中理论上包括6000条记录，在这里我们提取器中的R\G\B三种属性作为属性值，同时提取(nominal/anomaly)作为Ground Truth，进行数据点异常的判断。选取的方法包括：KNN / LOF / PCA / IForest。<jupyter_code># KNN
ROC_mean = []
rank_n_mean = []
time_c_total = []
begin_time = time()
roc = []
rank_n =[]
for i in range(total):
    data = pd.read_csv(benchmark_dir[i])
    seqset = data[["R","G","B"]]
    labelset = data["ground.truth"]
    seq = seqset.values
    label = np.where(labelset == "nominal" , 0 ,1)
    seq,label = shuffle(seq,label)
    X_train = seq[:int(train_rate*len(seq))]
    X_test = seq[int(train_rate*len(seq)):]
    Y_train = label[:int(train_rate*len(seq))]
    Y_test = label[int(train_rate*len(seq)):]
    if np.sum(Y_test) < 5:
        continue
    clf = KNN(n_neighbors = 5) 
    clf.fit(X_train) # 使用seq 训练检测器clf
    y_test_pred = clf.predict(X_test) 
    y_test_scores = clf.decision_function(X_test) 
    
    y = column_or_1d(Y_test)
    y_pred = column_or_1d(y_test_scores)
    check_consistent_length(y, y_pred)
    roc_ = np.round(roc_auc_score(y, y_pred), decimals=4)
    prn_ = np.round(precision_n_scores(y, y_pred), decimals=4)
    if i % 200 == 0:
        print("Current Iter "+str(i) ,": " , roc_)
    
    roc.append(roc_)
    rank_n.append(prn_)
end_time = time()
run_time = end_time-begin_time
ROC_mean.append(np.mean(roc))
rank_n_mean.append(np.mean(rank_n))
time_c_total.append(run_time)
# LOF
begin_time = time()
roc = []
rank_n =[]
for i in range(total):
    data = pd.read_csv(benchmark_dir[i])
    seqset = data[["R","G","B"]]
    labelset = data["ground.truth"]
    seq = seqset.values
    label = np.where(labelset == "nominal" , 0 ,1)
    seq,label = shuffle(seq,label)
    X_train = seq[:int(train_rate*len(seq))]
    X_test = seq[int(train_rate*len(seq)):]
    Y_train = label[:int(train_rate*len(seq))]
    Y_test = label[int(train_rate*len(seq)):]
    if np.sum(Y_test) < 5:
        continue
    clf = LOF() 
    clf.fit(X_train) # 使用seq 训练检测器clf
    y_test_scores = clf.decision_function(X_test) 
    
    y = column_or_1d(Y_test)
    y_pred = column_or_1d(y_test_scores)
    check_consistent_length(y, y_pred)
    roc_ = np.round(roc_auc_score(y, y_pred), decimals=4)
    prn_ = np.round(precision_n_scores(y, y_pred), decimals=4)
    if i % 200 == 0:
        print("Current Iter "+str(i) ,": " , roc_)
    
    roc.append(roc_)
    rank_n.append(prn_)
end_time = time()
run_time = end_time-begin_time
ROC_mean.append(np.mean(roc))
rank_n_mean.append(np.mean(rank_n))
time_c_total.append(run_time)
# PCA
begin_time = time()
roc = []
rank_n =[]
for i in range(total):
    data = pd.read_csv(benchmark_dir[i])
    seqset = data[["R","G","B"]]
    labelset = data["ground.truth"]
    seq = seqset.values
    label = np.where(labelset == "nominal" , 0 ,1)
    seq,label = shuffle(seq,label)
    X_train = seq[:int(train_rate*len(seq))]
    X_test = seq[int(train_rate*len(seq)):]
    Y_train = label[:int(train_rate*len(seq))]
    Y_test = label[int(train_rate*len(seq)):]
    if np.sum(Y_test) < 5:
        continue
    clf = PCA() 
    clf.fit(X_train) # 使用seq 训练检测器clf
    y_test_pred = clf.predict(X_test) 
    y_test_scores = clf.decision_function(X_test) 
    
    y = column_or_1d(Y_test)
    y_pred = column_or_1d(y_test_scores)
    check_consistent_length(y, y_pred)
    roc_ = np.round(roc_auc_score(y, y_pred), decimals=4)
    prn_ = np.round(precision_n_scores(y, y_pred), decimals=4)
    if i % 200 == 0:
        print("Current Iter "+str(i) ,": " , roc_)
        
    roc.append(roc_)
    rank_n.append(prn_)
end_time = time()
run_time = end_time-begin_time
ROC_mean.append(np.mean(roc))
rank_n_mean.append(np.mean(rank_n))
time_c_total.append(run_time)
# IForest
begin_time = time()
roc = []
rank_n =[]
for i in range(total):
    data = pd.read_csv(benchmark_dir[i])
    seqset = data[["R","G","B"]]
    labelset = data["ground.truth"]
    seq = seqset.values
    label = np.where(labelset == "nominal" , 0 ,1)
    seq,label = shuffle(seq,label)
    X_train = seq[:int(train_rate*len(seq))]
    X_test = seq[int(train_rate*len(seq)):]
    Y_train = label[:int(train_rate*len(seq))]
    Y_test = label[int(train_rate*len(seq)):]
    if np.sum(Y_test) < 5:
        continue
    clf = IForest(n_estimators = 50) 
    clf.fit(X_train) # 使用seq 训练检测器clf
    y_test_pred = clf.predict(X_test) 
    y_test_scores = clf.decision_function(X_test) 
    
    y = column_or_1d(Y_test)
    y_pred = column_or_1d(y_test_scores)
    check_consistent_length(y, y_pred)
    roc_ = np.round(roc_auc_score(y, y_pred), decimals=4)
    prn_ = np.round(precision_n_scores(y, y_pred), decimals=4)
    if i % 200 == 0:
        print("Current Iter "+str(i) ,": " , roc_)
    
    roc.append(roc_)
    rank_n.append(prn_)
end_time = time()
run_time = end_time-begin_time
ROC_mean.append(np.mean(roc))
rank_n_mean.append(np.mean(rank_n))
time_c_total.append(run_time)
plt.style.use('ggplot')
plt.figure(figsize=(9,6))
ROC = np.array(ROC_mean)
rank_n = np.array(rank_n_mean)
name = ["KNN" , "LOF" , "PCA","IForest"]
perfomance = np.array([ROC , rank_n])
for i in range(4):
    plt.scatter(rank_n[i] ,ROC[i] , s = 120 ,marker = "*")
plt.xlabel("rank_n")
plt.ylabel("ROC")
# plt.grid()
plt.legend(name)
plt.show()

plt.figure(figsize=(9,6))
plt.bar(name, time_c_total ,width=0.3)
plt.xlabel("Method")
plt.ylabel("Time consuming")
# plt.grid()
plt.show()<jupyter_output><empty_output><jupyter_text>从上面尝试的及几种方法来看，总体来说KNN与IForest的表现结果最好，相应的其计算异常点的消耗时间也更长，而PCA的表现分类结果不好，这里我认为可能是因为Skin的属性仅仅是三维数据，因而PCA主要用于降维并不能有效地进行异常点监测的缘故。 __2. 数据集分析 - pageb_benchmarks:__   该数据集中包含多条pageb_point的数据记录，数据中给出了pageb所对应的十维属性数值，并且由ground.truth属性提供了每一条记录点是否为异常点的记录。由上面的数据读取可以看出，在实验中使用了该数据总计包括940个benchmark，每个benchmark中理论上包括4466条记录，在这里我们提取器中的多种属性作为属性值，同时提取(nominal/anomaly)作为Ground Truth，进行数据点异常的判断。选取的方法依旧为：KNN / LOF / PCA / IForest。<jupyter_code>benchmark_dir1 = "F:\\course\\web\\homework4\\homework4\\pageb\\benchmarks\\"
benchmark_dir = []
for files in os.listdir(benchmark_dir1):
    file_path = os.path.join(benchmark_dir1, files)
    benchmark_dir.append(file_path)
np.random.shuffle(benchmark_dir)
train_rate = 0.7
total = len(benchmark_dir)
print("Benchmark number : " , total)
print("Example : ", benchmark_dir[0])
data = pd.read_csv(benchmark_dir[0])
print("Benchmark size:", data.shape)
data.head()
# KNN
ROC_mean = []
rank_n_mean = []
time_c_total = []
begin_time = time()
roc = []
rank_n =[]
for i in range(total):
    data = pd.read_csv(benchmark_dir[i])
    seqset = data[["V","V.1","V.2","V.3","V.4","V.5","V.6","V.7","V.8","V.9"]]
    labelset = data["ground.truth"]
    seq = seqset.values
    label = np.where(labelset == "nominal" , 0 ,1)
    seq,label = shuffle(seq,label)
    X_train = seq[:int(train_rate*len(seq))]
    X_test = seq[int(train_rate*len(seq)):]
    Y_train = label[:int(train_rate*len(seq))]
    Y_test = label[int(train_rate*len(seq)):]
    if np.sum(Y_test) < 5:
        continue
    clf = KNN(n_neighbors = 5) 
    clf.fit(X_train) # 使用seq 训练检测器clf
    y_test_pred = clf.predict(X_test) 
    y_test_scores = clf.decision_function(X_test) 
    
    y = column_or_1d(Y_test)
    y_pred = column_or_1d(y_test_scores)
    check_consistent_length(y, y_pred)
    roc_ = np.round(roc_auc_score(y, y_pred), decimals=4)
    prn_ = np.round(precision_n_scores(y, y_pred), decimals=4)
    if i % 200 == 0:
        print("Current Iter "+str(i) ,": " , roc_)
    
    roc.append(roc_)
    rank_n.append(prn_)
end_time = time()
run_time = end_time-begin_time
ROC_mean.append(np.mean(roc))
rank_n_mean.append(np.mean(rank_n))
time_c_total.append(run_time)
# LOF
begin_time = time()
roc = []
rank_n =[]
for i in range(total):
    data = pd.read_csv(benchmark_dir[i])
    seqset = data[["V","V.1","V.2","V.3","V.4","V.5","V.6","V.7","V.8","V.9"]]
    labelset = data["ground.truth"]
    seq = seqset.values
    label = np.where(labelset == "nominal" , 0 ,1)
    seq,label = shuffle(seq,label)
    X_train = seq[:int(train_rate*len(seq))]
    X_test = seq[int(train_rate*len(seq)):]
    Y_train = label[:int(train_rate*len(seq))]
    Y_test = label[int(train_rate*len(seq)):]
    if np.sum(Y_test) < 5:
        continue
    clf = LOF() 
    clf.fit(X_train) # 使用seq 训练检测器clf
    y_test_pred = clf.predict(X_test) 
    y_test_scores = clf.decision_function(X_test) 
    
    y = column_or_1d(Y_test)
    y_pred = column_or_1d(y_test_scores)
    check_consistent_length(y, y_pred)
    roc_ = np.round(roc_auc_score(y, y_pred), decimals=4)
    prn_ = np.round(precision_n_scores(y, y_pred), decimals=4)
    if i % 200 == 0:
        print("Current Iter "+str(i) ,": " , roc_)
    
    roc.append(roc_)
    rank_n.append(prn_)
end_time = time()
run_time = end_time-begin_time
ROC_mean.append(np.mean(roc))
rank_n_mean.append(np.mean(rank_n))
time_c_total.append(run_time)
# PCA
begin_time = time()
roc = []
rank_n =[]
for i in range(total):
    data = pd.read_csv(benchmark_dir[i])
    seqset = data[["V","V.1","V.2","V.3","V.4","V.5","V.6","V.7","V.8","V.9"]]
    labelset = data["ground.truth"]
    seq = seqset.values
    label = np.where(labelset == "nominal" , 0 ,1)
    seq,label = shuffle(seq,label)
    X_train = seq[:int(train_rate*len(seq))]
    X_test = seq[int(train_rate*len(seq)):]
    Y_train = label[:int(train_rate*len(seq))]
    Y_test = label[int(train_rate*len(seq)):]
    if np.sum(Y_test) < 5:
        continue
    clf = PCA() 
    clf.fit(X_train) # 使用seq 训练检测器clf
    y_test_pred = clf.predict(X_test) 
    y_test_scores = clf.decision_function(X_test) 
    
    y = column_or_1d(Y_test)
    y_pred = column_or_1d(y_test_scores)
    check_consistent_length(y, y_pred)
    roc_ = np.round(roc_auc_score(y, y_pred), decimals=4)
    prn_ = np.round(precision_n_scores(y, y_pred), decimals=4)
    if i % 200 == 0:
        print("Current Iter "+str(i) ,": " , roc_)
    
    roc.append(roc_)
    rank_n.append(prn_)
end_time = time()
run_time = end_time-begin_time
ROC_mean.append(np.mean(roc))
rank_n_mean.append(np.mean(rank_n))
time_c_total.append(run_time)
# IForest
begin_time = time()
roc = []
rank_n =[]
for i in range(total):
    data = pd.read_csv(benchmark_dir[i])
    seqset = data[["V","V.1","V.2","V.3","V.4","V.5","V.6","V.7","V.8","V.9"]]
    labelset = data["ground.truth"]
    seq = seqset.values
    label = np.where(labelset == "nominal" , 0 ,1)
    seq,label = shuffle(seq,label)
    X_train = seq[:int(train_rate*len(seq))]
    X_test = seq[int(train_rate*len(seq)):]
    Y_train = label[:int(train_rate*len(seq))]
    Y_test = label[int(train_rate*len(seq)):]
    if np.sum(Y_test) < 5:
        continue
    clf = IForest(n_estimators = 100) 
    clf.fit(X_train) # 使用seq 训练检测器clf
    y_test_pred = clf.predict(X_test) 
    y_test_scores = clf.decision_function(X_test) 
    
    y = column_or_1d(Y_test)
    y_pred = column_or_1d(y_test_scores)
    check_consistent_length(y, y_pred)
    roc_ = np.round(roc_auc_score(y, y_pred), decimals=4)
    prn_ = np.round(precision_n_scores(y, y_pred), decimals=4)
    if i % 200 == 0:
        print("Current Iter "+str(i) ,": " , roc_)
    
    roc.append(roc_)
    rank_n.append(prn_)
end_time = time()
run_time = end_time-begin_time
ROC_mean.append(np.mean(roc))
rank_n_mean.append(np.mean(rank_n))
time_c_total.append(run_time)
plt.style.use('ggplot')
plt.figure(figsize=(9,6))
ROC = np.array(ROC_mean)
rank_n = np.array(rank_n_mean)
name = ["KNN" , "LOF" , "PCA","IForest"]
perfomance = np.array([ROC , rank_n])
for i in range(4):
    plt.scatter(rank_n[i] ,ROC[i] , s = 120 ,marker = "*")
plt.xlabel("rank_n")
plt.ylabel("ROC")
# plt.grid()
plt.legend(name)
plt.show()

plt.figure(figsize=(9,6))
plt.bar(name, time_c_total ,width=0.3)
plt.xlabel("Method")
plt.ylabel("Time consuming")
# plt.grid()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1566383752652319, 'fraction_numerical': 0.016292816004849955, 'mean_word_length': 4.304260450160772, 'pattern_counts': {'":': 8, '<': 24, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '623234', 'license': 'MIT', 'n_tokens_mistral': 6009, 'n_tokens_neox': 5862, 'n_words': 828, 'path': 'Data_Homeworkw4.ipynb', 'repo_name': 'FadedFate/Data_Homeworkw4', 'size': '13196'}