<jupyter_start><jupyter_text>KDD Cup 1999 Data http://kdd.ics.uci.edu/databases/kddcup99/kddcup99.html<jupyter_code>import pandas as pd
import matplotlib.pyplot as pyplot
from sklearn import datasets
import sklearn.preprocessing as sp
from sklearn.externals import joblib
from sklearn import svm
% matplotlib inline<jupyter_output><empty_output><jupyter_text>|ファイル名|ファイル内容||---|---||kddcup.data|フルデータ||kddcup.data_10_percent|フルデータの10%を抽出した学習用データ||corrected|正常・攻撃のラベル付けがなされた評価用データ||kddcup.testdata.unlabeled|正常・攻撃のラベル付けがなされていないデータ||kddcup.testdata.unlabeled_10_percent|正常・攻撃のラベル付けがなされていないデータの10%サブセット||kddcup.newtestdata_10_percent_unlabeled|正常・攻撃のラベル付けがなされていないデータの10%サブセット|<jupyter_code>col_names = ["duration","protocol_type","service","flag","src_bytes",
             "dst_bytes","land","wrong_fragment","urgent","hot","num_failed_logins",
             "logged_in","num_compromised","root_shell","su_attempted","num_root","num_file_creations",
             "num_shells","num_access_files","num_outbound_cmds","is_host_login","is_guest_login","count",
             "srv_count","serror_rate","srv_serror_rate","rerror_rate","srv_rerror_rate","same_srv_rate",
             "diff_srv_rate","srv_diff_host_rate","dst_host_count","dst_host_srv_count",
             "dst_host_same_srv_rate","dst_host_diff_srv_rate","dst_host_same_src_port_rate",
             "dst_host_srv_diff_host_rate","dst_host_serror_rate","dst_host_srv_serror_rate",
             "dst_host_rerror_rate","dst_host_srv_rerror_rate","label"]
data = pd.read_csv("kddcup.data", header=None, names = col_names)
data.protocol_type = data.protocol_type.astype("category")
data.service = data.service.astype("category")
data.flag = data.flag.astype("category")
data.dtypes
le_protocol_type=sp.LabelEncoder()
le_protocol_type.fit(data.protocol_type)
#joblib.dump(le_protocol_type, 'dump/kddcup.data/le_protocol_type.pkl') 
data.protocol_type.value_counts()
data.protocol_type=le_protocol_type.transform(data.protocol_type) 
data.protocol_type.value_counts()
le_service=sp.LabelEncoder()
le_service.fit(data.service)
#joblib.dump(le_service, 'dump/kddcup.data/le_service.pkl') 
data.service.value_counts()
data.service=le_service.transform(data.service) 
data.service.value_counts()
le_flag=sp.LabelEncoder()
le_flag.fit(data.flag)
#joblib.dump(le_flag, 'dump/kddcup.data/le_flag.pkl') 
data.flag.value_counts()
data.flag=le_flag.transform(data.flag) 
data.flag.value_counts()
#joblib.dump(data, 'dump/kddcup.data/kddcup.data.00.pkl') 
le_label=sp.LabelEncoder()
le_label.fit(data.label)
#joblib.dump(le_label, 'dump/kddcup.data/le_label.pkl') 
data.label.value_counts()
data.label=le_label.transform(data.label) 
#joblib.dump(data, 'dump/kddcup.data/kddcup.data.01.pkl') 
kdd_data_10percent = pd.read_csv("kddcup.data_10_percent", header=None, names = col_names)
kdd_data_10percent.protocol_type.value_counts()
kdd_data_10percent.protocol_type=le_protocol_type.transform(kdd_data_10percent.protocol_type)
kdd_data_10percent.protocol_type.value_counts()
kdd_data_10percent.service.value_counts()
kdd_data_10percent.service=le_service.transform(kdd_data_10percent.service)
kdd_data_10percent.service.value_counts()
kdd_data_10percent.flag.value_counts()
kdd_data_10percent.flag=le_flag.transform(kdd_data_10percent.flag)
kdd_data_10percent.flag.value_counts()
#joblib.dump(kdd_data_10percent, 'dump/kddcup.data/kdd_data_10percent.00.pkl') 
kdd_data_10percent.label.value_counts()
kdd_data_10percent.label=le_label.transform(kdd_data_10percent.label)
kdd_data_10percent.label.value_counts()
#joblib.dump(kdd_data_10percent, 'dump/kddcup.data/kdd_data_10percent.01.pkl') 
train_features = kdd_data_10percent.drop('label',axis=1)
train_labels = kdd_data_10percent['label'].copy()
train_features.head()
train_labels.head()
le_label.inverse_transform(train_labels.head())
clf = svm.SVC()
clf.fit(train_features, train_labels)
#joblib.dump(clf, 'dump/kddcup.data/kdd_data_10percent.clf.pkl') 
test_pred = clf.predict(train_features)
from sklearn.metrics import classification_report, accuracy_score

print(classification_report(train_labels, test_pred))
print(accuracy_score(train_labels, test_pred))
train_features = kdd_data_10percent.drop('label',axis=1)
train_labels = kdd_data_10percent['label'].copy()
train_features= train_features.drop('service',axis=1)
clf2 = svm.SVC()
clf2.fit(train_features, train_labels)
#joblib.dump(clf2, 'dump/kddcup.data/kdd_data_10percent.clf2.pkl') 
test_pred = clf2.predict(train_features)
from sklearn.metrics import classification_report, accuracy_score

print(classification_report(train_labels, test_pred))
print(accuracy_score(train_labels, test_pred))
kdd_data_10percent.label = list(le_label.inverse_transform(kdd_data_10percent.label))
train_labels = kdd_data_10percent['label'].copy()
clf3 = svm.SVC()
clf3.fit(train_features, train_labels)
#joblib.dump(clf3, 'dump/kddcup.data/kdd_data_10percent.clf3.pkl') 
test_pred = clf3.predict(train_features)
from sklearn.metrics import classification_report, accuracy_score

print(classification_report(train_labels, test_pred))
print(accuracy_score(train_labels, test_pred))
corrected = pd.read_csv("corrected", header=None, names = col_names)
corrected.protocol_type=le_protocol_type.transform(corrected.protocol_type)
corrected.flag = le_flag.transform(corrected.flag)
test_features = corrected.drop(['label','service'],axis=1)
test_features.head()
test_labels = corrected.label.copy()
test_pred = clf3.predict(test_features)
from sklearn.metrics import classification_report, accuracy_score

print(classification_report(test_labels, test_pred))
print(accuracy_score(test_labels, test_pred))<jupyter_output>/Users/mhaya/.pyenv/versions/anaconda3-4.3.0/lib/python3.6/site-packages/sklearn/metrics/classification.py:1113: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/Users/mhaya/.pyenv/versions/anaconda3-4.3.0/lib/python3.6/site-packages/sklearn/metrics/classification.py:1115: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
{'fraction_non_alphanumeric': 0.12661498708010335, 'fraction_numerical': 0.021156330749354007, 'mean_word_length': 8.631415241057542, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '142867', 'license': 'MIT', 'n_tokens_mistral': 2449, 'n_tokens_neox': 2351, 'n_words': 238, 'path': 'KDDCUP99_01.ipynb', 'repo_name': 'mhaya/kddcup99', 'size': '6192'}