<commit_before>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ch `05`: Concept `03`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Self-organizing map"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import TensorFlow and NumPy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define a class called `SOM`. The constructor builds a grid of nodes, and also defines some helper ops:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class SOM:\n",
    "    \n",
    "    def __init__(self, width, height, dim):\n",
    "        self.num_iters = 100\n",
    "        self.width = width\n",
    "        self.height = height\n",
    "        self.dim = dim\n",
    "        self.node_locs = self.get_locs()\n",
    "\n",
    "        # Each node is a vector of dimension `dim`\n",
    "        # For a 2D grid, there are `width * height` nodes\n",
    "        nodes = tf.Variable(tf.random_normal([width*height, dim]))\n",
    "        self.nodes = nodes\n",
    "\n",
    "        # These two ops are inputs at each iteration\n",
    "        x = tf.placeholder(tf.float32, [dim])\n",
    "        iter = tf.placeholder(tf.float32)\n",
    "\n",
    "        self.x = x\n",
    "        self.iter = iter\n",
    "\n",
    "        # Find the node that matches closest to the input\n",
    "        bmu_loc = self.get_bmu_loc(x)\n",
    "\n",
    "        self.propagate_nodes = self.get_propagation(bmu_loc, x, iter)\n",
    "        \n",
    "    def get_propagation(self, bmu_loc, x, iter):\n",
    "        '''\n",
    "        Define the weight propagation function that will update weights of the best matching unit (BMU). \n",
    "        The intensity of weight updates decreases over time, as dictated by the `iter` variable.\n",
    "        '''\n",
    "        num_nodes = self.width * self.height\n",
    "        rate = 1.0 - tf.div(iter, self.num_iters)\n",
    "        alpha = rate * 0.5\n",
    "        sigma = rate * tf.to_float(tf.maximum(self.width, self.height)) / 2.\n",
    "        expanded_bmu_loc = tf.expand_dims(tf.to_float(bmu_loc), 0)\n",
    "        sqr_dists_from_bmu = tf.reduce_sum(tf.square(tf.sub(expanded_bmu_loc, self.node_locs)), 1)\n",
    "        neigh_factor = tf.exp(-tf.div(sqr_dists_from_bmu, 2 * tf.square(sigma)))\n",
    "        rate = tf.mul(alpha, neigh_factor)\n",
    "        rate_factor = tf.pack([tf.tile(tf.slice(rate, [i], [1]), [self.dim]) for i in range(num_nodes)])\n",
    "        nodes_diff = tf.mul(rate_factor, tf.sub(tf.pack([x for i in range(num_nodes)]), self.nodes))\n",
    "        update_nodes = tf.add(self.nodes, nodes_diff)\n",
    "        return tf.assign(self.nodes, update_nodes)\n",
    "    \n",
    "    def get_bmu_loc(self, x):\n",
    "        '''\n",
    "        Define a helper function to located the BMU:\n",
    "        '''\n",
    "        expanded_x = tf.expand_dims(x, 0)\n",
    "        sqr_diff = tf.square(tf.sub(expanded_x, self.nodes))\n",
    "        dists = tf.reduce_sum(sqr_diff, 1)\n",
    "        bmu_idx = tf.argmin(dists, 0)\n",
    "        bmu_loc = tf.pack([tf.mod(bmu_idx, self.width), tf.div(bmu_idx, self.width)])\n",
    "        return bmu_loc\n",
    "    \n",
    "    def get_locs(self):\n",
    "        '''\n",
    "        Build a grid of nodes:\n",
    "        '''\n",
    "        locs = [[x, y]\n",
    "            for y in range(self.height)\n",
    "            for x in range(self.width)]\n",
    "        return tf.to_float(locs)\n",
    "    \n",
    "    def train(self, data):\n",
    "        '''\n",
    "        Define a function to training the SOM on a given dataset:\n",
    "        '''\n",
    "        with tf.Session() as sess:\n",
    "            sess.run(tf.initialize_all_variables())\n",
    "            for i in range(self.num_iters):\n",
    "                for data_x in data:\n",
    "                    sess.run(self.propagate_nodes, feed_dict={self.x: data_x, self.iter: i})\n",
    "            centroid_grid = [[] for i in range(self.width)]\n",
    "            self.nodes_val = list(sess.run(self.nodes))\n",
    "            self.locs_val = list(sess.run(self.node_locs))\n",
    "            for i, l in enumerate(self.locs_val):\n",
    "                centroid_grid[int(l[0])].append(self.nodes_val[i])\n",
    "            self.centroid_grid = centroid_grid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Time to use our newfound powers. Let's test it out on some data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "<KEY>",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2727644f10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "colors = np.array(\n",
    "     [[0., 0., 1.],\n",
    "      [0., 0., 0.95],\n",
    "      [0., 0.05, 1.],\n",
    "      [0., 1., 0.],\n",
    "      [0., 0.95, 0.],\n",
    "      [0., 1, 0.05],\n",
    "      [1., 0., 0.],\n",
    "      [1., 0.05, 0.],\n",
    "      [1., 0., 0.05],\n",
    "      [1., 1., 0.]])\n",
    "\n",
    "som = SOM(4, 4, 3)\n",
    "som.train(colors)\n",
    "\n",
    "plt.imshow(som.centroid_grid)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
<commit_msg>updated SOM to TensorFlow v1<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Ch `05`: Concept `03`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Self-organizing map"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Import TensorFlow and NumPy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Define a class called `SOM`. The constructor builds a grid of nodes, and also defines some helper ops:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "class SOM:\n",
    "    \n",
    "    def __init__(self, width, height, dim):\n",
    "        self.num_iters = 100\n",
    "        self.width = width\n",
    "        self.height = height\n",
    "        self.dim = dim\n",
    "        self.node_locs = self.get_locs()\n",
    "\n",
    "        # Each node is a vector of dimension `dim`\n",
    "        # For a 2D grid, there are `width * height` nodes\n",
    "        nodes = tf.Variable(tf.random_normal([width*height, dim]))\n",
    "        self.nodes = nodes\n",
    "\n",
    "        # These two ops are inputs at each iteration\n",
    "        x = tf.placeholder(tf.float32, [dim])\n",
    "        iter = tf.placeholder(tf.float32)\n",
    "\n",
    "        self.x = x\n",
    "        self.iter = iter\n",
    "\n",
    "        # Find the node that matches closest to the input\n",
    "        bmu_loc = self.get_bmu_loc(x)\n",
    "\n",
    "        self.propagate_nodes = self.get_propagation(bmu_loc, x, iter)\n",
    "        \n",
    "    def get_propagation(self, bmu_loc, x, iter):\n",
    "        '''\n",
    "        Define the weight propagation function that will update weights of the best matching unit (BMU). \n",
    "        The intensity of weight updates decreases over time, as dictated by the `iter` variable.\n",
    "        '''\n",
    "        num_nodes = self.width * self.height\n",
    "        rate = 1.0 - tf.div(iter, self.num_iters)\n",
    "        alpha = rate * 0.5\n",
    "        sigma = rate * tf.to_float(tf.maximum(self.width, self.height)) / 2.\n",
    "        expanded_bmu_loc = tf.expand_dims(tf.to_float(bmu_loc), 0)\n",
    "        sqr_dists_from_bmu = tf.reduce_sum(tf.square(tf.subtract(expanded_bmu_loc, self.node_locs)), 1)\n",
    "        neigh_factor = tf.exp(-tf.div(sqr_dists_from_bmu, 2 * tf.square(sigma)))\n",
    "        rate = tf.multiply(alpha, neigh_factor)\n",
    "        rate_factor = tf.stack([tf.tile(tf.slice(rate, [i], [1]), [self.dim]) for i in range(num_nodes)])\n",
    "        nodes_diff = tf.multiply(rate_factor, tf.subtract(tf.stack([x for i in range(num_nodes)]), self.nodes))\n",
    "        update_nodes = tf.add(self.nodes, nodes_diff)\n",
    "        return tf.assign(self.nodes, update_nodes)\n",
    "    \n",
    "    def get_bmu_loc(self, x):\n",
    "        '''\n",
    "        Define a helper function to located the BMU:\n",
    "        '''\n",
    "        expanded_x = tf.expand_dims(x, 0)\n",
    "        sqr_diff = tf.square(tf.subtract(expanded_x, self.nodes))\n",
    "        dists = tf.reduce_sum(sqr_diff, 1)\n",
    "        bmu_idx = tf.argmin(dists, 0)\n",
    "        bmu_loc = tf.stack([tf.mod(bmu_idx, self.width), tf.div(bmu_idx, self.width)])\n",
    "        return bmu_loc\n",
    "    \n",
    "    def get_locs(self):\n",
    "        '''\n",
    "        Build a grid of nodes:\n",
    "        '''\n",
    "        locs = [[x, y]\n",
    "            for y in range(self.height)\n",
    "            for x in range(self.width)]\n",
    "        return tf.to_float(locs)\n",
    "    \n",
    "    def train(self, data):\n",
    "        '''\n",
    "        Define a function to training the SOM on a given dataset:\n",
    "        '''\n",
    "        with tf.Session() as sess:\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            for i in range(self.num_iters):\n",
    "                for data_x in data:\n",
    "                    sess.run(self.propagate_nodes, feed_dict={self.x: data_x, self.iter: i})\n",
    "            centroid_grid = [[] for i in range(self.width)]\n",
    "            self.nodes_val = list(sess.run(self.nodes))\n",
    "            self.locs_val = list(sess.run(self.node_locs))\n",
    "            for i, l in enumerate(self.locs_val):\n",
    "                centroid_grid[int(l[0])].append(self.nodes_val[i])\n",
    "            self.centroid_grid = centroid_grid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Time to use our newfound powers. Let's test it out on some data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "<KEY>",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f514b45fc50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "colors = np.array(\n",
    "     [[0., 0., 1.],\n",
    "      [0., 0., 0.95],\n",
    "      [0., 0.05, 1.],\n",
    "      [0., 1., 0.],\n",
    "      [0., 0.95, 0.],\n",
    "      [0., 1, 0.05],\n",
    "      [1., 0., 0.],\n",
    "      [1., 0.05, 0.],\n",
    "      [1., 0., 0.05],\n",
    "      [1., 1., 0.]])\n",
    "\n",
    "som = SOM(4, 4, 3)\n",
    "som.train(colors)\n",
    "\n",
    "plt.imshow(som.centroid_grid)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}

{'fraction_non_alphanumeric': 0.19626168224299065, 'fraction_numerical': 0.014675214335367266, 'mean_word_length': 1.884384049899755, 'pattern_counts': {'":': 128, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6700849', 'n_tokens_mistral': 5078, 'n_tokens_neox': 4885, 'n_words': 960}