<jupyter_start><jupyter_text>Classification pipeline<jupyter_code># Relative path, no need to change
script_path = '../script/'

# Absolute path, change it according to data locations
data_path = '../../data/'
feature_path = '../../features/'
# Get starting frame timestamps for all the clips
import glob
import pickle
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

import sklearn as sk
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import classification_report
from sklearn import metrics
import math<jupyter_output><empty_output><jupyter_text>Basic Supervised ML (not finished yet)<jupyter_code>def nan2zero(vec):
    return [0 if math.isnan(k) else k for k in vec]

def get_auc(test, pred):
#     print(test, pred)
    fpr, tpr, thresholds = metrics.roc_curve(test, pred)
#     print(fpr, tpr)
    return str(metrics.auc(fpr, tpr))

def evaluate_column(column):
    X = []
    y = []
    result = column + ','
#     print('=====================================')
#     print(column + ' results')
#     print('=====================================')
    for door in glob.glob('fridge_door*.pcap.csv'):
        data = pd.read_csv(door)
        X.append(nan2zero(vectorize(data, column)))
        y.append(1)

    for idle in glob.glob('fridge_idle*.pcap.csv'):
        data = pd.read_csv(idle)
        X.append(nan2zero(vectorize(data, column)))
        y.append(-1)

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33)

    LR = LogisticRegression(random_state=0, solver='lbfgs', multi_class='multinomial').fit(X_train, y_train)
    y_pred = LR.predict(X_test)
#     print('Logistic Regression:\n')
#     print(classification_report(y_test, y_pred))
    result += get_auc(y_test, y_pred) + ','

    SVM = svm.SVC(decision_function_shape="ovo").fit(X_train, y_train)
    y_pred = SVM.predict(X_test)
#     print('SVM:\n')
#     print(classification_report(y_test, y_pred))
    result += get_auc(y_test, y_pred) + ','

    RF = RandomForestClassifier(n_estimators=1000, max_depth=10, random_state=0).fit(X_train, y_train)
    y_pred = RF.predict(X_test)
#     print('Random Forest:\n')
#     print(classification_report(y_test, y_pred))
    result += get_auc(y_test, y_pred) + ','

    NN = MLPClassifier(solver='lbfgs', alpha=1e-5, hidden_layer_sizes=(150, 10), random_state=1).fit(X_train, y_train)
    y_pred = NN.predict(X_test)
#     print('Neural Network:\n')
#     print(classification_report(y_test, y_pred))
    result += get_auc(y_test, y_pred)
    
    print(result)
    return result
# import warnings filter
from warnings import simplefilter
# ignore all future warnings
simplefilter(action='ignore', category=FutureWarning)

# results = open("result.txt", "a")
for i in range(10):
    print(i)
    columns = ['frame.number', 'frame.time', 'frame.time_delta', 'frame.time_relative', 'eth.src_resolved', 'eth.dst_resolved', 'eth.src', 'eth.dst', 'eth.type', 'ip.version', 'ip.hdr_len', 'ip.len', 'ip.id', 'ip.flags.df', 'ip.flags.mf', 'ip.flags.rb', 'ip.flags.sf', 'ip.dsfield.dscp', 'ip.dsfield.ecn', 'ip.tos', 'ip.ttl', 'ip.proto', 'ip.src', 'ip.dst', 'udp.srcport', 'udp.dstport', 'udp.length', 'tcp.flags.cwr', 'tcp.flags.ecn', 'tcp.flags.urg', 'tcp.flags.ack', 'tcp.flags.push', 'tcp.flags.reset', 'tcp.flags.syn', 'tcp.flags.fin', 'tcp.flags.res', 'tcp.flags.ns', 'tcp.len', 'frame.len']
    for col in columns:
        if type(data[col].tolist()[1]).__name__ == 'float':
            evaluate_column(col)
# import warnings filter
from warnings import simplefilter
# ignore all future warnings
simplefilter(action='ignore', category=FutureWarning)

# results = open("result.txt", "a")
columns = ['frame.number', 'frame.time', 'frame.time_delta', 'frame.time_relative', 'eth.src_resolved', 'eth.dst_resolved', 'eth.src', 'eth.dst', 'eth.type', 'ip.version', 'ip.hdr_len', 'ip.len', 'ip.id', 'ip.flags.df', 'ip.flags.mf', 'ip.flags.rb', 'ip.flags.sf', 'ip.dsfield.dscp', 'ip.dsfield.ecn', 'ip.tos', 'ip.ttl', 'ip.proto', 'ip.src', 'ip.dst', 'udp.srcport', 'udp.dstport', 'udp.length', 'tcp.flags.cwr', 'tcp.flags.ecn', 'tcp.flags.urg', 'tcp.flags.ack', 'tcp.flags.push', 'tcp.flags.reset', 'tcp.flags.syn', 'tcp.flags.fin', 'tcp.flags.res', 'tcp.flags.ns', 'tcp.len', 'frame.len']
for col in columns:
    if type(data[col].tolist()[1]).__name__ == 'float':
        evaluate_column(col)
# import warnings filter
from warnings import simplefilter
# ignore all future warnings
simplefilter(action='ignore', category=FutureWarning)

# results = open("result.txt", "a")
columns = ['frame.number', 'frame.time', 'frame.time_delta', 'frame.time_relative', 'eth.src_resolved', 'eth.dst_resolved', 'eth.src', 'eth.dst', 'eth.type', 'ip.version', 'ip.hdr_len', 'ip.len', 'ip.id', 'ip.flags.df', 'ip.flags.mf', 'ip.flags.rb', 'ip.flags.sf', 'ip.dsfield.dscp', 'ip.dsfield.ecn', 'ip.tos', 'ip.ttl', 'ip.proto', 'ip.src', 'ip.dst', 'udp.srcport', 'udp.dstport', 'udp.length', 'tcp.flags.cwr', 'tcp.flags.ecn', 'tcp.flags.urg', 'tcp.flags.ack', 'tcp.flags.push', 'tcp.flags.reset', 'tcp.flags.syn', 'tcp.flags.fin', 'tcp.flags.res', 'tcp.flags.ns', 'tcp.len', 'frame.len']
for col in columns:
    if type(data[col].tolist()[1]).__name__ == 'float':
        evaluate_column(col)
#         results.write(evaluate_column(col))
for col in data.columns:
    if type(data[col].tolist()[1]).__name__ == 'float':
        evaluate_column(col)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19181494661921708, 'fraction_numerical': 0.005516014234875445, 'mean_word_length': 4.632264529058117, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '630620', 'license': 'Apache-2.0', 'n_tokens_mistral': 2052, 'n_tokens_neox': 2017, 'n_words': 410, 'path': 'legacy/Classification Pipeline & Result Interpretation.ipynb', 'repo_name': 'Learn-Live/activity_recognition', 'size': '5620'}