<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from graphtime import GroupFusedGraphLasso, IndepFusedGraphLasso\n",
    "from graphtime.utils import get_edges, get_change_points, plot_data_with_cps\n",
    "from graphtime.utils import precision, recall, f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "plt.rcParams['figure.figsize'] = (10.0, 7.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y = np.load('data/y.npy')\n",
    "sigma = np.load('data/sigma.npy')\n",
    "sigma_inv = np.load('data/sigma_inv.npy')\n",
    "T = 90 # Steps\n",
    "K = 2 # Changepoints\n",
    "P = 10 # Variables\n",
    "M = 5 # Active Edges\n",
    "eps = 0.000001 # Edge threshold epsilon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "edges = get_edges(sigma_inv[0], eps, P)\n",
    "y_true = get_change_points(sigma_inv, eps, T, P)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run Estimation with GFGL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "verbose = False\n",
    "tol = 1e-4\n",
    "max_iter = 500\n",
    "gammas = [1, 1, 1] # gamma_V1, gamma_V2, gamma_W\n",
    "lambda1 = 0.15\n",
    "lambda2 = 25\n",
    "gfgl = GroupFusedGraphLasso(lambda1, lambda2, gammas[0], gammas[1], gammas[2], tol, max_iter, verbose)\n",
    "gfgl.fit(y);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F, Precision, Recall\n",
    "According to [this paper](https://projecteuclid.org/download/pdfview_1/euclid.ejs/1351865118), we treat graph\n",
    "recovery as a classification problem of edges. For the mathematical definition, see paper p.19."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.11178451178451154"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision(gfgl.sparse_Theta, sigma_inv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall(gfgl.sparse_Theta, sigma_inv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.20109024833434244"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(gfgl.sparse_Theta, sigma_inv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exhaustive Grid Search with performance estimation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def performance(G_est, G_true):\n",
    "    prec = precision(G_est, G_true)\n",
    "    rec = recall(G_est, G_true)\n",
    "    f1 = f1_score(G_est, G_true)\n",
    "    return prec, rec, f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def grid_search(y, G_true, lam1s, lam2s, tol=1e-4, max_iter=500, gamma1=1, gamma2=1, gammaw=1):\n",
    "    performances = dict()\n",
    "    for lam1 in lam1s:\n",
    "        for lam2 in lam2s:\n",
    "            gfgl = GroupFusedGraphLasso(lam1, lam2, gamma1, gamma2, gammaw, tol, max_iter)\n",
    "            gfgl.fit(y)\n",
    "            performances[(lam1, lam2)] = performance(gfgl.sparse_Theta, G_true)\n",
    "    return performances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lam1s = [0.1, 0.15, 0.2]\n",
    "lam2s = [20, 25, 30]\n",
    "pfs = grid_search(y, sigma_inv, lam1s, lam2s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(0.1, 20): (0.1111111111111109, 1.0, 0.19999999999999965),\n",
       " (0.1, 25): (0.11156004489337801, 1.0, 0.20072697899838413),\n",
       " (0.1, 30): (0.1111111111111109, 1.0, 0.19999999999999965),\n",
       " (0.15, 20): (0.11136363636363614, 1.0, 0.20040899795500985),\n",
       " (0.15, 25): (0.11178451178451154, 1.0, 0.20109024833434244),\n",
       " (0.15, 30): (0.1111111111111109, 1.0, 0.19999999999999965),\n",
       " (0.2, 20): (0.11136363636363614, 1.0, 0.20040899795500985),\n",
       " (0.2, 25): (0.11178451178451154, 1.0, 0.20109024833434244),\n",
       " (0.2, 30): (0.11189674523007832, 1.0, 0.20127182800040336)}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },<commit_msg>Performance estimation example<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from graphtime import GroupFusedGraphLasso, IndepFusedGraphLasso\n",
    "from graphtime.utils import get_edges, get_change_points, plot_data_with_cps\n",
    "from graphtime.utils import precision, recall, f1_score, grid_search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "plt.rcParams['figure.figsize'] = (10.0, 7.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y = np.load('data/y.npy')\n",
    "sigma = np.load('data/sigma.npy')\n",
    "sigma_inv = np.load('data/sigma_inv.npy')\n",
    "T = 90 # Steps\n",
    "K = 2 # Changepoints\n",
    "P = 10 # Variables\n",
    "M = 5 # Active Edges\n",
    "eps = 0.000001 # Edge threshold epsilon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "edges = get_edges(sigma_inv[0], eps, P)\n",
    "y_true = get_change_points(sigma_inv, eps, T, P)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run Estimation with GFGL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "verbose = False\n",
    "tol = 1e-4\n",
    "max_iter = 500\n",
    "gammas = [1, 1, 1] # gamma_V1, gamma_V2, gamma_W\n",
    "lambda1 = 0.15\n",
    "lambda2 = 25\n",
    "gfgl = GroupFusedGraphLasso(lambda1, lambda2, gammas[0], gammas[1], gammas[2], tol, max_iter, verbose)\n",
    "gfgl.fit(y);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F, Precision, Recall\n",
    "According to [this paper](https://projecteuclid.org/download/pdfview_1/euclid.ejs/1351865118), we treat graph\n",
    "recovery as a classification problem of edges. For the mathematical definition, see paper p.19."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.11178451178451154"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision(gfgl.sparse_Theta, sigma_inv, per_ts=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall(gfgl.sparse_Theta, sigma_inv, per_ts=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.20109024833434244"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(gfgl.sparse_Theta, sigma_inv, per_ts=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exhaustive Grid Search with performance estimation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lams = np.logspace(0, 11, base=2, num=10)\n",
    "pfs = grid_search(GroupFusedGraphLasso, y, sigma_inv, lams, lams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "f1s = {v[2]:k for k,v in pfs.items()}\n",
    "prc = {v[0]:k for k,v in pfs.items()}\n",
    "rec = {v[1]:k for k,v in pfs.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6666666666666666"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(prc.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.0, 2048.0)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1s[0.5238095238095243]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
{'fraction_non_alphanumeric': 0.2022359419322543, 'fraction_numerical': 0.060153512431169696, 'mean_word_length': 2.437625466016633, 'pattern_counts': {'":': 261, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 7, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3799247', 'n_tokens_mistral': 5297, 'n_tokens_neox': 4766, 'n_words': 827}