<commit_before><commit_msg>add basic scaled KNN model<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# KDD Cup 1999 \n",
    "# http://kdd.ics.uci.edu/databases/kddcup99/kddcup99.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Table 1: Basic features of individual TCP connections.\n",
    "|feature name | description | type |\n",
    "|-------------|-------------|------|\n",
    "|duration |length (number of seconds) of the connection | continuous |\n",
    "|protocol_type |type of the protocol, e.g. tcp, udp, etc. | discrete |\n",
    "|service | network service on the destination, e.g., http, telnet, etc. | discrete |\n",
    "|src_bytes | number of data bytes from source to destination | continuous |\n",
    "|dst_bytes | number of data bytes from destination to source | continuous |\n",
    "|flag | normal or error status of the connection | discrete |\n",
    "|land | 1 if connection is from/to the same host/port; 0 otherwise | discrete |\n",
    "|wrong_fragment | number of ''wrong'' fragments | continuous |\n",
    "|urgent | number of urgent packets | continuous |"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Table 2: Content features within a connection suggested by domain knowledge.\n",
    "| feature name\t| description | type |\n",
    "|---------------|-------------|------|\n",
    "| hot | number of ''hot'' indicators | continuous |\n",
    "| num_failed_logins | number of failed login attempts | continuous |\n",
    "| logged_in | 1 if successfully logged in; 0 otherwise | discrete |\n",
    "| num_compromised | number of ''compromised'' conditions | continuous |\n",
    "| root_shell | 1 if root shell is obtained; 0 otherwise | discrete |\n",
    "| su_attempted | 1 if ''su root'' command attempted; 0 otherwise | discrete |\n",
    "| num_root | number of ''root'' accesses | continuous |\n",
    "| num_file_creations | number of file creation operations | continuous |\n",
    "| num_shells | number of shell prompts | continuous |\n",
    "| num_access_files | number of operations on access control files | continuous |\n",
    "| num_outbound_cmds\t| number of outbound commands in an ftp session | continuous |\n",
    "| is_hot_login | 1 if the login belongs to the ''hot'' list; 0 otherwise | discrete |\n",
    "| is_guest_login | 1 if the login is a ''guest''login; 0 otherwise | discrete |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "#%matplotlib notebook\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as mpatches\n",
    "import warnings\n",
    "import constants\n",
    "import utils\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data_10_percent = 'kddcup.data_10_percent'\n",
    "data_full = 'kddcup.data'\n",
    "data = pd.read_csv(data_10_percent, names=constants.names)\n",
    "\n",
    "# Remove Traffic features computed using a two-second time window\n",
    "data.drop(constants.traffic_features, inplace=True, axis=1)\n",
    "\n",
    "# Categorical features to numeric labels\n",
    "\n",
    "from sklearn import preprocessing\n",
    "\n",
    "le_dicts = {}\n",
    "\n",
    "for categorical_name in constants.categorical_names:\n",
    "    le = preprocessing.LabelEncoder()\n",
    "    le.fit(data[categorical_name])\n",
    "    le_dicts[categorical_name] = dict(zip(le.transform(le.classes_), le.classes_))\n",
    "#     print(categorical_name, ':', le_dicts[categorical_name])\n",
    "    data[categorical_name + '_num'] = le.fit_transform(data[categorical_name])\n",
    "\n",
    "data['label_binary_num'] = data.label.apply(lambda label: 0 if label == 'normal.' else 1)\n",
    "data['label_four'] = data.label.apply(lambda label: constants.label_to_four_attack_class[label])\n",
    "data['label_four_num'] = data.label_four.apply(lambda label: constants.five_classes_to_num[label])\n",
    "data.drop(constants.categorical_names + ['label', 'label_four'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Normalize data\n",
    "from sklearn import preprocessing\n",
    "\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "data_transform = min_max_scaler.fit_transform(data[constants.names_to_normalize])\n",
    "data_scaled = data.copy(deep=True)\n",
    "data_scaled[constants.names_to_normalize] = pd.DataFrame(data_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#test-train division\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test = train_test_split(data, test_size=0.1, random_state=42)\n",
    "\n",
    "Y_train_bin = X_train.label_binary_num\n",
    "Y_train_many_classes = X_train.label_four_num\n",
    "\n",
    "Y_test_bin = X_test.label_binary_num\n",
    "Y_test_many_classes = X_test.label_four_num\n",
    "\n",
    "X_train.drop(['label_binary_num', 'label_four_num'], axis=1, inplace=True)\n",
    "X_test.drop(['label_binary_num', 'label_four_num'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#test-train division scaled\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train_scaled, X_test_scaled = train_test_split(data_scaled, test_size=0.1, random_state=42)\n",
    "\n",
    "X_train_scaled.drop(['label_binary_num', 'label_four_num'], axis=1, inplace=True)\n",
    "X_test_scaled.drop(['label_binary_num', 'label_four_num'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\n",
      "0.196885866069\n",
      "[[ 87538      0]\n",
      " [357079      1]]\n",
      "Test:\n",
      "0.197154018987\n",
      "[[ 9740     0]\n",
      " [39663     0]]\n"
     ]
    }
   ],
   "source": [
    "#KMeans for not scaled data\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score\n",
    "\n",
    "kmeans = KMeans(n_clusters=2,\n",
    "                max_iter=500,\n",
    "                random_state=42)\n",
    "\n",
    "utils.model_validation(kmeans, X_train, X_test, Y_train_bin, Y_test_bin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#KMeans for scaled data\n",
    "kmeans = KMeans(n_clusters=2,\n",
    "                max_iter=500,\n",
    "                random_state=42)\n",
    "utils.model_validation(kmeans, X_train_scaled, X_test_scaled, Y_train_bin, Y_test_bin)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

{'fraction_non_alphanumeric': 0.18140301915514398, 'fraction_numerical': 0.015476341494354942, 'mean_word_length': 3.0122137404580154, 'pattern_counts': {'":': 75, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3454849', 'n_tokens_mistral': 2905, 'n_tokens_neox': 2788, 'n_words': 592}