<commit_before>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Meta Reinforcement Learning with A3C - Rainbow Gridworld\n",
    "\n",
    "This iPython notebook includes an implementation of the [A3C algorithm capable of Meta-RL](https://arxiv.org/pdf/1611.05763.pdf).\n",
    "\n",
    "For more information see the accompanying [Medium post](https://medium.com/p/b15b592a2ddf)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import threading\n",
    "import multiprocessing\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import tensorflow.contrib.slim as slim\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "from PIL import Image\n",
    "from PIL import ImageDraw \n",
    "from PIL import ImageFont\n",
    "\n",
    "import scipy.signal\n",
    "%matplotlib inline\n",
    "from helper import *\n",
    "\n",
    "from random import choice\n",
    "from time import sleep\n",
    "from time import time\n",
    "from gridworld import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Actor-Critic Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class AC_Network():\n",
    "    def __init__(self,a_size,scope,trainer):\n",
    "        with tf.variable_scope(scope):\n",
    "            #Input and visual encoding layers\n",
    "            self.state = tf.placeholder(shape=[None,5,5,3],dtype=tf.float32)\n",
    "            self.conv = slim.fully_connected(slim.flatten(self.state),64,activation_fn=tf.nn.elu)\n",
    "            self.prev_rewards = tf.placeholder(shape=[None,1],dtype=tf.float32)\n",
    "            self.prev_actions = tf.placeholder(shape=[None],dtype=tf.int32)\n",
    "            self.timestep = tf.placeholder(shape=[None,1],dtype=tf.float32)\n",
    "            self.prev_actions_onehot = tf.one_hot(self.prev_actions,a_size,dtype=tf.float32)\n",
    "\n",
    "            hidden = tf.concat(1,[slim.flatten(self.conv),self.prev_rewards,self.prev_actions_onehot,self.timestep])\n",
    "            \n",
    "            #Recurrent network for temporal dependencies\n",
    "            lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(48,state_is_tuple=True)\n",
    "            c_init = np.zeros((1, lstm_cell.state_size.c), np.float32)\n",
    "            h_init = np.zeros((1, lstm_cell.state_size.h), np.float32)\n",
    "            self.state_init = [c_init, h_init]\n",
    "            c_in = tf.placeholder(tf.float32, [1, lstm_cell.state_size.c])\n",
    "            h_in = tf.placeholder(tf.float32, [1, lstm_cell.state_size.h])\n",
    "            self.state_in = (c_in, h_in)\n",
    "            rnn_in = tf.expand_dims(hidden, [0])\n",
    "            step_size = tf.shape(self.prev_rewards)[:1]\n",
    "            state_in = tf.nn.rnn_cell.LSTMStateTuple(c_in, h_in)\n",
    "            lstm_outputs, lstm_state = tf.nn.dynamic_rnn(\n",
    "                lstm_cell, rnn_in, initial_state=state_in, sequence_length=step_size,\n",
    "                time_major=False)\n",
    "            lstm_c, lstm_h = lstm_state\n",
    "            self.state_out = (lstm_c[:1, :], lstm_h[:1, :])\n",
    "            rnn_out = tf.reshape(lstm_outputs, [-1, 48])\n",
    "            \n",
    "            self.actions = tf.placeholder(shape=[None],dtype=tf.int32)\n",
    "            self.actions_onehot = tf.one_hot(self.actions,a_size,dtype=tf.float32)\n",
    "                        \n",
    "            #Output layers for policy and value estimations\n",
    "            self.policy = slim.fully_connected(rnn_out,a_size,\n",
    "                activation_fn=tf.nn.softmax,\n",
    "                weights_initializer=normalized_columns_initializer(0.01),\n",
    "                biases_initializer=None)\n",
    "            self.value = slim.fully_connected(rnn_out,1,\n",
    "                activation_fn=None,\n",
    "                weights_initializer=normalized_columns_initializer(1.0),\n",
    "                biases_initializer=None)\n",
    "            \n",
    "            #Only the worker network need ops for loss functions and gradient updating.\n",
    "            if scope != 'global':\n",
    "                self.target_v = tf.placeholder(shape=[None],dtype=tf.float32)\n",
    "                self.advantages = tf.placeholder(shape=[None],dtype=tf.float32)\n",
    "                \n",
    "                self.responsible_outputs = tf.reduce_sum(self.policy * self.actions_onehot, [1])\n",
    "\n",<commit_msg>TF 1.0 compatibility<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Meta Reinforcement Learning with A3C - Rainbow Gridworld\n",
    "\n",
    "This iPython notebook includes an implementation of the [A3C algorithm capable of Meta-RL](https://arxiv.org/pdf/1611.05763.pdf).\n",
    "\n",
    "For more information see the accompanying [Medium post](https://medium.com/p/b15b592a2ddf)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import threading\n",
    "import multiprocessing\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import tensorflow.contrib.slim as slim\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "from PIL import Image\n",
    "from PIL import ImageDraw \n",
    "from PIL import ImageFont\n",
    "\n",
    "import scipy.signal\n",
    "%matplotlib inline\n",
    "from helper import *\n",
    "\n",
    "from random import choice\n",
    "from time import sleep\n",
    "from time import time\n",
    "from gridworld import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Actor-Critic Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class AC_Network():\n",
    "    def __init__(self,a_size,scope,trainer):\n",
    "        with tf.variable_scope(scope):\n",
    "            #Input and visual encoding layers\n",
    "            self.state = tf.placeholder(shape=[None,5,5,3],dtype=tf.float32)\n",
    "            self.conv = slim.fully_connected(slim.flatten(self.state),64,activation_fn=tf.nn.elu)\n",
    "            self.prev_rewards = tf.placeholder(shape=[None,1],dtype=tf.float32)\n",
    "            self.prev_actions = tf.placeholder(shape=[None],dtype=tf.int32)\n",
    "            self.timestep = tf.placeholder(shape=[None,1],dtype=tf.float32)\n",
    "            self.prev_actions_onehot = tf.one_hot(self.prev_actions,a_size,dtype=tf.float32)\n",
    "\n",
    "            hidden = tf.concat(1,[slim.flatten(self.conv),self.prev_rewards,self.prev_actions_onehot,self.timestep])\n",
    "            \n",
    "            #Recurrent network for temporal dependencies\n",
    "            lstm_cell = tf.contrib.rnn.BasicLSTMCell(48,state_is_tuple=True)\n",
    "            c_init = np.zeros((1, lstm_cell.state_size.c), np.float32)\n",
    "            h_init = np.zeros((1, lstm_cell.state_size.h), np.float32)\n",
    "            self.state_init = [c_init, h_init]\n",
    "            c_in = tf.placeholder(tf.float32, [1, lstm_cell.state_size.c])\n",
    "            h_in = tf.placeholder(tf.float32, [1, lstm_cell.state_size.h])\n",
    "            self.state_in = (c_in, h_in)\n",
    "            rnn_in = tf.expand_dims(hidden, [0])\n",
    "            step_size = tf.shape(self.prev_rewards)[:1]\n",
    "            state_in = tf.contrib.rnn.LSTMStateTuple(c_in, h_in)\n",
    "            lstm_outputs, lstm_state = tf.nn.dynamic_rnn(\n",
    "                lstm_cell, rnn_in, initial_state=state_in, sequence_length=step_size,\n",
    "                time_major=False)\n",
    "            lstm_c, lstm_h = lstm_state\n",
    "            self.state_out = (lstm_c[:1, :], lstm_h[:1, :])\n",
    "            rnn_out = tf.reshape(lstm_outputs, [-1, 48])\n",
    "            \n",
    "            self.actions = tf.placeholder(shape=[None],dtype=tf.int32)\n",
    "            self.actions_onehot = tf.one_hot(self.actions,a_size,dtype=tf.float32)\n",
    "                        \n",
    "            #Output layers for policy and value estimations\n",
    "            self.policy = slim.fully_connected(rnn_out,a_size,\n",
    "                activation_fn=tf.nn.softmax,\n",
    "                weights_initializer=normalized_columns_initializer(0.01),\n",
    "                biases_initializer=None)\n",
    "            self.value = slim.fully_connected(rnn_out,1,\n",
    "                activation_fn=None,\n",
    "                weights_initializer=normalized_columns_initializer(1.0),\n",
    "                biases_initializer=None)\n",
    "            \n",
    "            #Only the worker network need ops for loss functions and gradient updating.\n",
    "            if scope != 'global':\n",
    "                self.target_v = tf.placeholder(shape=[None],dtype=tf.float32)\n",
    "                self.advantages = tf.placeholder(shape=[None],dtype=tf.float32)\n",
    "                \n",
    "                self.responsible_outputs = tf.reduce_sum(self.policy * self.actions_onehot, [1])\n",
    "\n",
{'fraction_non_alphanumeric': 0.16115573172003755, 'fraction_numerical': 0.015020340043809324, 'mean_word_length': 2.172733289212442, 'pattern_counts': {'":': 38, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6467884', 'n_tokens_mistral': 3438, 'n_tokens_neox': 3334, 'n_words': 509}