<commit_before><commit_msg>lecture01<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 546,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib as mpl\n",
    "import networkx as nx\n",
    "import visJS2jupyter.visJS_module\n",
    "import visJS2jupyter.visualizations as visualizations\n",
    "\n",
    "G = nx.Graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 547,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# nodes\n",
    "\n",
    "# topics\n",
    "topics = [\"Approximation Theory\",\n",
    "          \"Harmonic Analysis\",\n",
    "          \"Neuroscience\",\n",
    "          \"Statistics/Machine Learning\",\n",
    "          \"Optimization\"]\n",
    "G.add_nodes_from(topics)\n",
    "\n",
    "# questions\n",
    "questions = [\"Overfitting\",\n",
    "             \"Regularization\",\n",
    "             \"Implicit regularization\",\n",
    "             \"Generalization\",\n",
    "             \"Curse of dimensionality\",\n",
    "             \"Deep versus shallow\",\n",
    "             \"Landscape of loss\",\n",
    "             \"Biological plausibility\",\n",
    "             \"Robustness to noise\",\n",
    "             \"Stability to deformations\",\n",
    "             \"Energy propagation\",\n",
    "             \"Translation invariance\",\n",
    "             \"Invariance to intraclass variability\",\n",
    "             \"Generative modeling\",\n",
    "             \"Architectures as inference algorithms\",\n",
    "             \"Interpretation of tricks\",\n",
    "             \"Sparse coding\",\n",
    "             \"Network design\"]\n",
    "G.add_nodes_from(questions)\n",
    "\n",
    "# lectures in the stats385 course\n",
    "lectures = [\"Lecture 3:\\n<NAME>kei\",\n",
    "            \"Lecture 4:\\n<NAME>\",\n",
    "            \"Lecture 5:\\n<NAME>\"]\n",
    "G.add_nodes_from(lectures)\n",
    "\n",
    "nodes = G.nodes()\n",
    "\n",
    "nodes_shape = []\n",
    "node_shape = ['square' if (node in topics) else ('dot' if (node in questions) else 'star') for node in G.nodes()]\n",
    "node_to_shape = dict(zip(G.nodes(),node_shape))\n",
    "\n",
    "nodes_size = []\n",
    "node_size = [5 if (node in topics) else (4 if (node in questions) else 3) for node in G.nodes()]\n",
    "node_to_size = dict(zip(G.nodes(),node_size))\n",
    "\n",
    "node_to_color = []\n",
    "node_color = ['#800080' if (node in topics) else ('#C70039' if (node in questions) else '#FFC300') for node in G.nodes()]\n",
    "node_to_color = dict(zip(G.nodes(),node_color))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 548,
   "metadata": {},
   "outputs": [],
   "source": [
    "# edges\n",
    "\n",
    "# topics and questions\n",
    "topics_questions = [(\"Approximation Theory\",\"Curse of dimensionality\",{'title':\"How does deep learning overcome the curse of dimensionality?\"}),\n",
    "                    (\"Approximation Theory\",\"Deep versus shallow\",{'title':\"When and why are deep networks better than shallow networks?\"}),\n",
    "                    (\"Harmonic Analysis\",\"Translation invariance\",{'title':\"Are the features computed by deep architectures translation invariant or covariant?\"}),\n",
    "                    (\"Harmonic Analysis\",\"Stability to deformations\",{'title':\"Are the features stable to deformations?\"}),\n",
    "                    (\"Harmonic Analysis\",\"Energy propagation\",{'title':\"How does the energy of the features behave as function of the layer and can we tune it?\"}),\n",
    "                    (\"Harmonic Analysis\",\"Robustness to noise\",{'title':\"If noise is added to the input of the architecture, do the estimated feature representations remain stable?\"}),\n",
    "                    (\"Harmonic Analysis\",\"Invariance to intraclass variability\",{'title':\"Is the network learning invariance to certain variabilities?\"}),\n",
    "                    (\"Harmonic Analysis\",\"Network design\",{'title':\"How should we design the architecture?\"}),\n",
    "                    (\"Neuroscience\",\"Biological plausibility\",{'title':\"Are modern architectures biologically plausible?\"}),\n",
    "                    (\"Statistics/Machine Learning\",\"Overfitting\",{'title':\"How can deep learning not overfit?\"}),\n",
    "                    (\"Statistics/Machine Learning\",\"Regularization\",{'title':\"How does the network manage to generalize without regularization despite the large number of parameters?\"}),\n",
    "                    (\"Statistics/Machine Learning\",\"Implicit regularization\",{'title':\"Is there any implicit regularization in the network?\"}),\n",
    "                    (\"Statistics/Machine Learning\",\"Generalization\",{'title':\"What does the generalization capability of a network rely on?\"}),\n",
    "                    (\"Statistics/Machine Learning\",\"Generative modeling\",{'title':\"Is there a generative model behind deep learning architectures and if so what is it?\"}),\n",
    "                    (\"Statistics/Machine Learning\",\"Architectures as inference algorithms\",{'title':\"Can we interpret certain networks as an inference algorithm under a generative model?\"}),\n",
    "                    (\"Statistics/Machine Learning\",\"Interpretation of tricks\",{'title':\"What is the importance of different tricks?\"}),\n",
    "                    (\"Statistics/Machine Learning\",\"Sparse coding\",{'title':\"Why do the filters trained in the first layer resemble those obtained from sparse coding?\"}),\n",
    "                    (\"Optimization\",\"Landscape of loss\",{'title':\"What is the landscape of the objective function being minimized?\"})]\n",
    "G.add_edges_from(topics_questions)\n",
    "\n",
    "# lectures and questions\n",
    "lectures_questions = [(\"Lecture 3:\\n<NAME>\",\"Translation invariance\",{'title':\"Features become more translation invariant with increasing network depth.\"}),\n",
    "                      (\"Lecture 3:\\n<NAME>\",\"Stability to deformations\",{'title':\"For certain classes of signals, the features are stable to deformations.\"}),\n",
    "                      (\"Lecture 3:\\n<NAME>\",\"Energy propagation\",{'title':\"The energy of the features decays exponentially or polynomially, depending on the assumptions on the filters.\"}),\n",
    "                      (\"Lecture 3:\\n<NAME>\",\"Network design\",{'title':\"Filters or number of layers can be designed for energy preservation.\"}),\n",
    "                      (\"Lecture 3:\\n<NAME>\",\"Deep versus shallow\",{'title':\"Depth width tradeoff for networks with wavelet filters.\"}),\n",
    "                      (\"Lecture 4:\\nAnkit Patel\",\"Generative modeling\",{'title':\"NN-DRMM: a heirarchical generative deep sparse coding model.\"}),\n",
    "                      (\"Lecture 4:\\nAnkit Patel\",\"Architectures as inference algorithms\",{'title':\"Convnets are a max-sum-product message passing bottom-up inference algorithm.\"}),\n",
    "                      (\"Lecture 4:\\nAnkit Patel\",\"Interpretation of tricks\",{'title':\"ReLU and max pooling as max-marginalization.\"}),\n",
    "                      (\"Lecture 4:\\nAnkit Patel\",\"Sparse coding\",{'title':\"Only the sparse set of active paths matter for the final decision of the convnet.\"}),\n",
    "                      (\"Lecture 5:\\nTomaso Poggio\",\"Deep versus shallow\",{'title':\"For compositional functions deep networks avoid the curse of dimensionality because of locality of constituent functions.\"}),\n",
    "                      (\"Lecture 5:\\nTomaso Poggio\",\"Landscape of loss\",{'title':\"Many global minima that are found by SGD with high probability.\"}),\n",
    "                      (\"Lecture 5:\\nTomaso Poggio\",\"Overfitting\",{'title':\"Gradient descent avoids overfitting without explicit regularization, despite overparametrization.\"}),\n",
    "                      (\"Lecture 5:\\nTomaso Poggio\",\"Implicit regularization\",{'title':\"Gradient descent results in implicit regularization in deep linear networks.\"})]\n",
    "G.add_edges_from(lectures_questions)\n",
    "\n",
    "edges = G.edges(data=True)\n",
    "\n",
    "edge_to_color = []\n",
    "edge_color = ['gray' if (edge in topics_questions) else 'gray' for edge in G.edges()]\n",
    "edge_to_color = dict(zip(G.edges(),edge_color))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 549,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<!doctype html><html><head>  <title>Network | Basic usage</title></head><body><script type=\"text/javascript\">function setUpFrame() {     var frame = window.frames[\"style_file0\"];    frame.runVis([{\"id\": \"Approximation Theory\", \"color\": \"#800080\", \"node_shape\": \"square\", \"degree\": 5, \"x\": 566.9341948461183, \"y\": 11.255394639596682, \"border_width\": 0, \"title\": \"Approximation Theory\"}, {\"id\": \"Harmonic Analysis\", \"color\": \"#800080\", \"node_shape\": \"square\", \"degree\": 5, \"x\": 71.57309865956466, \"y\": 614.0129837591466, \"border_width\": 0, \"title\": \"Harmonic Analysis\"}, {\"id\": \"Neuroscience\", \"color\": \"#800080\", \"node_shape\": \"square\", \"degree\": 5, \"x\": 998.4581842955437, \"y\": 597.8510870957892, \"border_width\": 0, \"title\": \"Neuroscience\"}, {\"id\": \"Statistics/Machine Learning\", \"color\": \"#800080\", \"node_shape\": \"square\", \"degree\": 5, \"x\": 761.6749179817767, \"y\": 664.3973328648965, \"border_width\": 0, \"title\": \"Statistics/Machine Learning\"}, {\"id\": \"Optimization\", \"color\": \"#800080\", \"node_shape\": \"square\", \"degree\": 5, \"x\": 451.0384877772572, \"y\": 0.0, \"border_width\": 0, \"title\": \"Optimization\"}, {\"id\": \"Overfitting\", \"color\": \"#C70039\", \"node_shape\": \"dot\", \"degree\": 4, \"x\": 783.5360141258955, \"y\": 294.9249652888483, \"border_width\": 0, \"title\": \"Overfitting\"}, {\"id\": \"Regularization\", \"color\": \"#C70039\", \"node_shape\": \"dot\", \"degree\": 4, \"x\": 539.8733176430964, \"y\": 1000.0, \"border_width\": 0, \"title\": \"Regularization\"}, {\"id\": \"Implicit regularization\", \"color\": \"#C70039\", \"node_shape\": \"dot\", \"degree\": 4, \"x\": 610.2466833101669, \"y\": 417.5978192022881, \"border_width\": 0, \"title\": \"Implicit regularization\"}, {\"id\": \"Generalization\", \"color\": \"#C70039\", \"node_shape\": \"dot\", \"degree\": 4, \"x\": 666.1471427411471, \"y\": 976.1740572976032, \"border_width\": 0, \"title\": \"Generalization\"}, {\"id\": \"Curse of dimensionality\", \"color\": \"#C70039\", \"node_shape\": \"dot\", \"degree\": 4, \"x\": 756.1514068919614, \"y\": 75.24125588559411, \"border_width\": 0, \"title\": \"Curse of dimensionality\"}, {\"id\": \"Deep versus shallow\", \"color\": \"#C70039\", \"node_shape\": \"dot\", \"degree\": 4, \"x\": 231.68743427012066, \"y\": 117.97743832341521, \"border_width\": 0, \"title\": \"Deep versus shallow\"}, {\"id\": \"Landscape of loss\", \"color\": \"#C70039\", \"node_shape\": \"dot\", \"degree\": 4, \"x\": 338.22860327038956, \"y\": 27.32684672083554, \"border_width\": 0, \"title\": \"Landscape of loss\"}, {\"id\": \"Biological plausibility\", \"color\": \"#C70039\", \"node_shape\": \"dot\", \"degree\": 4, \"x\": 878.907182475557, \"y\": 838.1816279498956, \"border_width\": 0, \"title\": \"Biological plausibility\"}, {\"id\": \"Robustness to noise\", \"color\": \"#C70039\", \"node_shape\": \"dot\", \"degree\": 4, \"x\": 114.03169686069282, \"y\": 828.5629466474139, \"border_width\": 0, \"title\": \"Robustness to noise\"}, {\"id\": \"Stability to deformations\", \"color\": \"#C70039\", \"node_shape\": \"dot\", \"degree\": 4, \"x\": 0.0, \"y\": 544.4414329598902, \"border_width\": 0, \"title\": \"Stability to deformations\"}, {\"id\": \"Energy propagation\", \"color\": \"#C70039\", \"node_shape\": \"dot\", \"degree\": 4, \"x\": 35.49474124862943, \"y\": 694.0561583579014, \"border_width\": 0, \"title\": \"Energy propagation\"}, {\"id\": \"Translation invariance\", \"color\": \"#C70039\", \"node_shape\": \"dot\", \"degree\": 4, \"x\": 61.13617213658716, \"y\": 274.3934382410753, \"border_width\": 0, \"title\": \"Translation invariance\"}, {\"id\": \"Invariance to intraclass variability\", \"color\": \"#C70039\", \"node_shape\": \"dot\", \"degree\": 4, \"x\": 204.14641097669482, \"y\": 909.8607924523438, \"border_width\": 0, \"title\": \"Invariance to intraclass variability\"}, {\"id\": \"Generative modeling\", \"color\": \"#C70039\", \"node_shape\": \"dot\", \"degree\": 4, \"x\": 993.52210226152, \"y\": 407.522075744865, \"border_width\": 0, \"title\": \"Generative modeling\"}, {\"id\": \"Architectures as inference algorithms\", \"color\": \"#C70039\", \"node_shape\": \"dot\", \"degree\": 4, \"x\": 976.6459880839445, \"y\": 499.76002781124845, \"border_width\": 0, \"title\": \"Architectures as inference algorithms\"}, {\"id\": \"Interpretation of tricks\", \"color\": \"#C70039\", \"node_shape\": \"dot\", \"degree\": 4, \"x\": 931.3549045582629, \"y\": 766.7728427540884, \"border_width\": 0, \"title\": \"Interpretation of tricks\"}, {\"id\": \"Sparse coding\", \"color\": \"#C70039\", \"node_shape\": \"dot\", \"degree\": 4, \"x\": 800.4019743811543, \"y\": 905.1832792027211, \"border_width\": 0, \"title\": \"Sparse coding\"}, {\"id\": \"Network design\", \"color\": \"#C70039\", \"node_shape\": \"dot\", \"degree\": 4, \"x\": 23.73936711341998, \"y\": 353.04976995721245, \"border_width\": 0, \"title\": \"Network design\"}, {\"id\": \"Lecture 3:\\nHelmut Bolcskei\", \"color\": \"#FFC300\", \"node_shape\": \"star\", \"degree\": 3, \"x\": 13.626245654520506, \"y\": 419.7026361429474, \"border_width\": 0, \"title\": \"Lecture 3:\\n<NAME>\"}, {\"id\": \"Lecture 4:\\nAnkit Patel\", \"color\": \"#FFC300\", \"node_shape\": \"star\", \"degree\": 3, \"x\": 964.9260678396342, \"y\": 672.234140168771, \"border_width\": 0, \"title\": \"Lecture 4:\\nAnkit Patel\"}, {\"id\": \"Lecture 5:\\n<NAME>\", \"color\": \"#FFC300\", \"node_shape\": \"star\", \"degree\": 3, \"x\": 504.5096778322089, \"y\": 174.0732305787653, \"border_width\": 0, \"title\": \"Lecture 5:\\nTomaso Poggio\"}], [{\"source\": 0, \"target\": 9, \"color\": \"gray\", \"title\": \"How does deep learning overcome the curse of dimensionality?\"}, {\"source\": 0, \"target\": 10, \"color\": \"gray\", \"title\": \"When and why are deep networks better than shallow networks?\"}, {\"source\": 1, \"target\": 16, \"color\": \"gray\", \"title\": \"Are the features computed by deep architectures translation invariant or covariant?\"}, {\"source\": 1, \"target\": 14, \"color\": \"gray\", \"title\": \"Are the features stable to deformations?\"}, {\"source\": 1, \"target\": 15, \"color\": \"gray\", \"title\": \"How does the energy of the features behave as function of the layer and can we tune it?\"}, {\"source\": 1, \"target\": 13, \"color\": \"gray\", \"title\": \"If noise is added to the input of the architecture, do the estimated feature representations remain stable?\"}, {\"source\": 1, \"target\": 17, \"color\": \"gray\", \"title\": \"Is the network learning invariance to certain variabilities?\"}, {\"source\": 1, \"target\": 22, \"color\": \"gray\", \"title\": \"How should we design the architecture?\"}, {\"source\": 2, \"target\": 12, \"color\": \"gray\", \"title\": \"Are modern architectures biologically plausible?\"}, {\"source\": 3, \"target\": 5, \"color\": \"gray\", \"title\": \"How can deep learning not overfit?\"}, {\"source\": 3, \"target\": 6, \"color\": \"gray\", \"title\": \"How does the network manage to generalize without regularization despite the large number of parameters?\"}, {\"source\": 3, \"target\": 7, \"color\": \"gray\", \"title\": \"Is there any implicit regularization in the network?\"}, {\"source\": 3, \"target\": 8, \"color\": \"gray\", \"title\": \"What does the generalization capability of a network rely on?\"}, {\"source\": 3, \"target\": 18, \"color\": \"gray\", \"title\": \"Is there a generative model behind deep learning architectures and if so what is it?\"}, {\"source\": 3, \"target\": 19, \"color\": \"gray\", \"title\": \"Can we interpret certain networks as an inference algorithm under a generative model?\"}, {\"source\": 3, \"target\": 20, \"color\": \"gray\", \"title\": \"What is the importance of different tricks?\"}, {\"source\": 3, \"target\": 21, \"color\": \"gray\", \"title\": \"Why do the filters trained in the first layer resemble those obtained from sparse coding?\"}, {\"source\": 4, \"target\": 11, \"color\": \"gray\", \"title\": \"What is the landscape of the objective function being minimized?\"}, {\"source\": 5, \"target\": 25, \"color\": \"gray\", \"title\": \"Gradient descent avoids overfitting without explicit regularization, despite overparametrization.\"}, {\"source\": 7, \"target\": 25, \"color\": \"gray\", \"title\": \"Gradient descent results in implicit regularization in deep linear networks.\"}, {\"source\": 10, \"target\": 23, \"color\": \"gray\", \"title\": \"Depth width tradeoff for networks with wavelet filters.\"}, {\"source\": 10, \"target\": 25, \"color\": \"gray\", \"title\": \"For compositional functions deep networks avoid the curse of dimensionality because of locality of constituent functions.\"}, {\"source\": 11, \"target\": 25, \"color\": \"gray\", \"title\": \"Many global minima that are found by SGD with high probability.\"}, {\"source\": 14, \"target\": 23, \"color\": \"gray\", \"title\": \"For certain classes of signals, the features are stable to deformations.\"}, {\"source\": 15, \"target\": 23, \"color\": \"gray\", \"title\": \"The energy of the features decays exponentially or polynomially, depending on the assumptions on the filters.\"}, {\"source\": 16, \"target\": 23, \"color\": \"gray\", \"title\": \"Features become more translation invariant with increasing network depth.\"}, {\"source\": 18, \"target\": 24, \"color\": \"gray\", \"title\": \"NN-DRMM: a heirarchical generative deep sparse coding model.\"}, {\"source\": 19, \"target\": 24, \"color\": \"gray\", \"title\": \"Convnets are a max-sum-product message passing bottom-up inference algorithm.\"}, {\"source\": 20, \"target\": 24, \"color\": \"gray\", \"title\": \"ReLU and max pooling as max-marginalization.\"}, {\"source\": 21, \"target\": 24, \"color\": \"gray\", \"title\": \"Only the sparse set of active paths matter for the final decision of the convnet.\"}, {\"source\": 22, \"target\": 23, \"color\": \"gray\", \"title\": \"Filters or number of layers can be designed for energy preservation.\"}]);}</script><iframe name=\"style_file0\" src=\"style_file0.html\" height=\"1200px\" width=\"100%;\"></iframe></body></html>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 549,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# set node initial positions using networkx's spring_layout function\n",
    "pos = nx.spring_layout(G)\n",
    "\n",
    "nodes_dict = [{\"id\":n,\n",
    "               \"color\":node_to_color[n],\n",
    "               \"node_shape\":node_to_shape[n],\n",
    "               \"degree\":node_to_size[n],\n",
    "               \"x\":pos[n][0]*1000,\n",
    "               \"y\":pos[n][1]*1000} for n in nodes]\n",
    "node_map = dict(zip(nodes,range(len(nodes))))  # map to indices for source/target in edges\n",
    "edges_dict = [{\"source\":node_map[edges[i][0]], \"target\":node_map[edges[i][1]], \n",
    "              \"color\":\"gray\",\n",
    "              \"title\":edges[i][2]['title']} for i in range(len(edges))]\n",
    "\n",
    "# set some network-wide styles\n",
    "visJS2jupyter.visJS_module.visjs_network(nodes_dict,edges_dict,\n",
    "                          node_size_multiplier=10,\n",
    "                          node_size_transform = '',\n",
    "                          node_color_highlight_border='black',\n",
    "                          node_color_highlight_background='#8BADD3',\n",
    "                          node_color_hover_border='blue',\n",
    "                          node_color_hover_background='#8BADD3',\n",
    "                          node_font_size=20,\n",
    "                          edge_arrow_to=False,\n",
    "                          physics_enabled=True,\n",
    "                          edge_color_highlight='#8BADD3',\n",
    "                          edge_color_hover='#8BADD3',\n",
    "                          edge_width=3,\n",
    "                          edge_title_field='title',\n",
    "                          max_velocity=15,\n",
    "                          min_velocity=1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
{'fraction_non_alphanumeric': 0.23764882347426644, 'fraction_numerical': 0.0556388862848036, 'mean_word_length': 2.8016749821810407, 'pattern_counts': {'":': 390, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7437144', 'n_tokens_mistral': 7422, 'n_tokens_neox': 6396, 'n_words': 1788}