<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from scipy.io import mmread\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "malware_classes = [\"Agent\", \"AutoRun\", \"FraudLoad\", \"FraudPack\", \"Hupigon\", \"Krap\",\n",
    "           \"Lipler\", \"Magania\", \"None\", \"Poison\", \"Swizzor\", \"Tdss\",\n",
    "           \"VB\", \"Virut\", \"Zbot\"]\n",
    "\n",
    "# a function for writing predictions in the required format\n",
    "def write_predictions(predictions, ids, outfile):\n",
    "    \"\"\"\n",
    "    assumes len(predictions) == len(ids), and that predictions[i] is the\n",
    "    index of the predicted class with the malware_classes list above for\n",
    "    the executable corresponding to ids[i].\n",
    "    outfile will be overwritten\n",
    "    \"\"\"\n",
    "    with open(outfile,\"w+\") as f:\n",
    "        # write header\n",
    "        f.write(\"Id,Prediction\\n\")\n",
    "        for i, history_id in enumerate(ids):\n",
    "            f.write(\"%s,%d\\n\" % (history_id, predictions[i]))\n",
    "\n",
    "def classes_to_Y(classes):\n",
    "    output = []\n",
    "    for cls in classes:\n",
    "        output.append(malware_classes.index(cls))\n",
    "    return np.array(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# load training classes\n",
    "classes = np.load(\"../data/features/train_classes.npy\")\n",
    "\n",
    "# load sparse matrix of training data\n",
    "sparse_mat_train_test = mmread(\"../data/features/naive_word_hashed_full_features.mtx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# convert csr to a numpy array\n",
    "sparse = sparse_mat_train_test.toarray()\n",
    "\n",
    "# pull out training examples\n",
    "X = sparse[:classes.shape[0]]\n",
    "# X_CV = X[-300:]\n",
    "# X = X[:-300]\n",
    "\n",
    "X_test = sparse[classes.shape[0]:]\n",
    "print X_test.shape\n",
    "\n",
    "Y = classes_to_Y(classes)\n",
    "# Y_CV = Y[-300:]\n",
    "# Y = Y[:-300]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "            n_estimators=10, n_jobs=1, oob_score=False, random_state=None,\n",
       "            verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RF = RandomForestClassifier()\n",
    "RF.fit(X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'RF' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-ed548c1bc0cd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_CV\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mmistakes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mpreds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mY_CV\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'RF' is not defined"
     ]
    }
   ],
   "source": [
    "# preds = RF.predict(X_CV)\n",
    "\n",
    "# mistakes = 0\n",
    "# for i in range(len(preds)):\n",
    "#     if preds[i] != Y_CV[i]:\n",
    "#         mistakes += 1\n",
    "#         print malware_classes[Y_CV[i]]\n",
    "# print mistakes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "test_pred = RF.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print test_pred\n",
    "test_ids = np.load(\"../data/features/test_ids.npy\")\n",
    "print test_ids\n",
    "write_predictions(test_pred, test_ids, \"../predictions/rfc_10.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}<commit_msg>commit for merge<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from scipy.io import mmread\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "malware_classes = [\"Agent\", \"AutoRun\", \"FraudLoad\", \"FraudPack\", \"Hupigon\", \"Krap\",\n",
    "           \"Lipler\", \"Magania\", \"None\", \"Poison\", \"Swizzor\", \"Tdss\",\n",
    "           \"VB\", \"Virut\", \"Zbot\"]\n",
    "\n",
    "# a function for writing predictions in the required format\n",
    "def write_predictions(predictions, ids, outfile):\n",
    "    \"\"\"\n",
    "    assumes len(predictions) == len(ids), and that predictions[i] is the\n",
    "    index of the predicted class with the malware_classes list above for\n",
    "    the executable corresponding to ids[i].\n",
    "    outfile will be overwritten\n",
    "    \"\"\"\n",
    "    with open(outfile,\"w+\") as f:\n",
    "        # write header\n",
    "        f.write(\"Id,Prediction\\n\")\n",
    "        for i, history_id in enumerate(ids):\n",
    "            f.write(\"%s,%d\\n\" % (history_id, predictions[i]))\n",
    "\n",
    "def classes_to_Y(classes):\n",
    "    output = []\n",
    "    for cls in classes:\n",
    "        output.append(malware_classes.index(cls))\n",
    "    return np.array(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# load training classes\n",
    "classes = np.load(\"../data/features/train_classes.npy\")\n",
    "\n",
    "# load sparse matrix of training data\n",
    "sparse_mat_train_test = mmread(\"../data/features/naive_word_hashed_full_features.mtx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# convert csr to a numpy array\n",
    "sparse = sparse_mat_train_test.toarray()\n",
    "\n",
    "# pull out training examples\n",
    "X = sparse[:classes.shape[0]]\n",
    "# X_CV = X[-300:]\n",
    "# X = X[:-300]\n",
    "\n",
    "X_test = sparse[classes.shape[0]:]\n",
    "print X_test.shape\n",
    "\n",
    "Y = classes_to_Y(classes)\n",
    "# Y_CV = Y[-300:]\n",
    "# Y = Y[:-300]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "            n_estimators=10, n_jobs=1, oob_score=False, random_state=None,\n",
       "            verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RF = RandomForestClassifier()\n",
    "RF.fit(X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'RF' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-ed548c1bc0cd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_CV\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mmistakes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mpreds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mY_CV\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'RF' is not defined"
     ]
    }
   ],
   "source": [
    "# preds = RF.predict(X_CV)\n",
    "\n",
    "# mistakes = 0\n",
    "# for i in range(len(preds)):\n",
    "#     if preds[i] != Y_CV[i]:\n",
    "#         mistakes += 1\n",
    "#         print malware_classes[Y_CV[i]]\n",
    "# print mistakes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'RF' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-b96bc3bf7410>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtest_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mprint\u001b[0m \u001b[0mtest_pred\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'RF' is not defined"
     ]
    }
   ],
   "source": [
    "test_pred = RF.predict(X_test[:1])\n",
    "print test_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# test_pred = RF.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# print test_pred\n",
    "test_ids = np.load(\"../data/features/test_ids.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# print test_ids\n",
    "write_predictions(test_pred, test_ids, \"../predictions/rfc_10.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
{'fraction_non_alphanumeric': 0.2227384531292703, 'fraction_numerical': 0.09162339437004646, 'mean_word_length': 2.5980825958702063, 'pattern_counts': {'":': 196, '<': 9, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5415989', 'n_tokens_mistral': 7094, 'n_tokens_neox': 6395, 'n_words': 830}