<commit_before>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Meta Reinforcement Learning with A3C\n",
    "\n",
    "This iPython notebook includes an implementation of the [A3C algorithm capable of Meta-RL](https://arxiv.org/pdf/1611.05763.pdf).\n",
    "\n",
    "For more information, see the accompanying [Medium post](https://medium.com/p/b15b592a2ddf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import threading\n",
    "import multiprocessing\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import tensorflow.contrib.slim as slim\n",
    "import scipy.signal\n",
    "from PIL import Image\n",
    "from PIL import ImageDraw \n",
    "from PIL import ImageFont\n",
    "%matplotlib inline\n",
    "from helper import *\n",
    "\n",
    "from random import choice\n",
    "from time import sleep\n",
    "from time import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class dependent_bandit():\n",
    "    def __init__(self,difficulty):\n",
    "        self.num_actions = 2\n",
    "        self.difficulty = difficulty\n",
    "        self.reset()\n",
    "        \n",
    "    def set_restless_prob(self):\n",
    "        self.bandit = np.array([self.restless_list[self.timestep],1 - self.restless_list[self.timestep]])\n",
    "        \n",
    "    def reset(self):\n",
    "        self.timestep = 0\n",
    "        if self.difficulty == 'restless': \n",
    "            variance = np.random.uniform(0,.5)\n",
    "            self.restless_list = np.cumsum(np.random.uniform(-variance,variance,(150,1)))\n",
    "            self.restless_list = (self.restless_list - np.min(self.restless_list)) / (np.max(self.restless_list - np.min(self.restless_list))) \n",
    "            self.set_restless_prob()\n",
    "        if self.difficulty == 'easy': bandit_prob = np.random.choice([0.9,0.1])\n",
    "        if self.difficulty == 'medium': bandit_prob = np.random.choice([0.75,0.25])\n",
    "        if self.difficulty == 'hard': bandit_prob = np.random.choice([0.6,0.4])\n",
    "        if self.difficulty == 'uniform': bandit_prob = np.random.uniform()\n",
    "        if self.difficulty != 'independent' and self.difficulty != 'restless':\n",
    "            self.bandit = np.array([bandit_prob,1 - bandit_prob])\n",
    "        else:\n",
    "            self.bandit = np.random.uniform(size=2)\n",
    "        \n",
    "    def pullArm(self,action):\n",
    "        #Get a random number.\n",
    "        if self.difficulty == 'restless': self.set_restless_prob()\n",
    "        self.timestep += 1\n",
    "        bandit = self.bandit[action]\n",
    "        result = np.random.uniform()\n",
    "        if result < bandit:\n",
    "            #return a positive reward.\n",
    "            reward = 1\n",
    "        else:\n",
    "            #return a negative reward.\n",
    "            reward = 0\n",
    "        if self.timestep > 99: \n",
    "            done = True\n",
    "        else: done = False\n",
    "        return reward,done,self.timestep"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Actor-Critic Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class AC_Network():\n",
    "    def __init__(self,a_size,scope,trainer):\n",
    "        with tf.variable_scope(scope):\n",
    "            #Input and visual encoding layers\n",
    "            self.prev_rewards = tf.placeholder(shape=[None,1],dtype=tf.float32)\n",
    "            self.prev_actions = tf.placeholder(shape=[None],dtype=tf.int32)\n",
    "            self.timestep = tf.placeholder(shape=[None,1],dtype=tf.float32)\n",
    "            self.prev_actions_onehot = tf.one_hot(self.prev_actions,a_size,dtype=tf.float32)\n",
    "\n",
    "            hidden = tf.concat(1,[self.prev_rewards,self.prev_actions_onehot,self.timestep])\n",
    "            \n",
    "            #Recurrent network for temporal dependencies\n",
    "            lstm_cell = tf.contrib.rnn.BasicLSTMCell(48,state_is_tuple=True)\n",
    "            c_init = np.zeros((1, lstm_cell.state_size.c), np.float32)\n",
    "            h_init = np.zeros((1, lstm_cell.state_size.h), np.float32)\n",
    "            self.state_init = [c_init, h_init]\n",
    "            c_in = tf.placeholder(tf.float32, [1, lstm_cell.state_size.c])\n",
    "            h_in = tf.placeholder(tf.float32, [1, lstm_cell.state_size.h])\n",
    "            self.state_in = (c_in, h_in)\n",
    "            rnn_in = tf.expand_dims(hidden, [0])\n",
    "            step_size = tf.shape(self.prev_rewards)[:1]\n",
    "            state_in = tf.contrib.rnn.LSTMStateTuple(c_in, h_in)\n",
    "            lstm_outputs, lstm_state = tf.nn.dynamic_rnn(\n",
    "                lstm_cell, rnn_in, initial_state=state_in, sequence_length=step_size,\n",
    "                time_major=False)\n",
    "            lstm_c, lstm_h = lstm_state\n",
    "            self.state_out = (lstm_c[:1, :], lstm_h[:1, :])\n",
    "            rnn_out = tf.reshape(lstm_outputs, [-1, 48])\n",
    "            \n",
    "            self.actions = tf.placeholder(shape=[None],dtype=tf.int32)\n",
    "            self.actions_onehot = tf.one_hot(self.actions,a_size,dtype=tf.float32)\n",
    "                        \n",
    "            #Output layers for policy and value estimations\n",
    "            self.policy = slim.fully_connected(rnn_out,a_size,\n",
    "                activation_fn=tf.nn.softmax,\n",
    "                weights_initializer=normalized_columns_initializer(0.01),\n",
    "                biases_initializer=None)\n",
    "            self.value = slim.fully_connected(rnn_out,1,\n",<commit_msg>TF 1.0 fix<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Meta Reinforcement Learning with A3C\n",
    "\n",
    "This iPython notebook includes an implementation of the [A3C algorithm capable of Meta-RL](https://arxiv.org/pdf/1611.05763.pdf).\n",
    "\n",
    "For more information, see the accompanying [Medium post](https://medium.com/p/b15b592a2ddf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import threading\n",
    "import multiprocessing\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import tensorflow.contrib.slim as slim\n",
    "import scipy.signal\n",
    "from PIL import Image\n",
    "from PIL import ImageDraw \n",
    "from PIL import ImageFont\n",
    "%matplotlib inline\n",
    "from helper import *\n",
    "\n",
    "from random import choice\n",
    "from time import sleep\n",
    "from time import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class dependent_bandit():\n",
    "    def __init__(self,difficulty):\n",
    "        self.num_actions = 2\n",
    "        self.difficulty = difficulty\n",
    "        self.reset()\n",
    "        \n",
    "    def set_restless_prob(self):\n",
    "        self.bandit = np.array([self.restless_list[self.timestep],1 - self.restless_list[self.timestep]])\n",
    "        \n",
    "    def reset(self):\n",
    "        self.timestep = 0\n",
    "        if self.difficulty == 'restless': \n",
    "            variance = np.random.uniform(0,.5)\n",
    "            self.restless_list = np.cumsum(np.random.uniform(-variance,variance,(150,1)))\n",
    "            self.restless_list = (self.restless_list - np.min(self.restless_list)) / (np.max(self.restless_list - np.min(self.restless_list))) \n",
    "            self.set_restless_prob()\n",
    "        if self.difficulty == 'easy': bandit_prob = np.random.choice([0.9,0.1])\n",
    "        if self.difficulty == 'medium': bandit_prob = np.random.choice([0.75,0.25])\n",
    "        if self.difficulty == 'hard': bandit_prob = np.random.choice([0.6,0.4])\n",
    "        if self.difficulty == 'uniform': bandit_prob = np.random.uniform()\n",
    "        if self.difficulty != 'independent' and self.difficulty != 'restless':\n",
    "            self.bandit = np.array([bandit_prob,1 - bandit_prob])\n",
    "        else:\n",
    "            self.bandit = np.random.uniform(size=2)\n",
    "        \n",
    "    def pullArm(self,action):\n",
    "        #Get a random number.\n",
    "        if self.difficulty == 'restless': self.set_restless_prob()\n",
    "        self.timestep += 1\n",
    "        bandit = self.bandit[action]\n",
    "        result = np.random.uniform()\n",
    "        if result < bandit:\n",
    "            #return a positive reward.\n",
    "            reward = 1\n",
    "        else:\n",
    "            #return a negative reward.\n",
    "            reward = 0\n",
    "        if self.timestep > 99: \n",
    "            done = True\n",
    "        else: done = False\n",
    "        return reward,done,self.timestep"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Actor-Critic Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class AC_Network():\n",
    "    def __init__(self,a_size,scope,trainer):\n",
    "        with tf.variable_scope(scope):\n",
    "            #Input and visual encoding layers\n",
    "            self.prev_rewards = tf.placeholder(shape=[None,1],dtype=tf.float32)\n",
    "            self.prev_actions = tf.placeholder(shape=[None],dtype=tf.int32)\n",
    "            self.timestep = tf.placeholder(shape=[None,1],dtype=tf.float32)\n",
    "            self.prev_actions_onehot = tf.one_hot(self.prev_actions,a_size,dtype=tf.float32)\n",
    "\n",
    "            hidden = tf.concat([self.prev_rewards,self.prev_actions_onehot,self.timestep],1)\n",
    "            \n",
    "            #Recurrent network for temporal dependencies\n",
    "            lstm_cell = tf.contrib.rnn.BasicLSTMCell(48,state_is_tuple=True)\n",
    "            c_init = np.zeros((1, lstm_cell.state_size.c), np.float32)\n",
    "            h_init = np.zeros((1, lstm_cell.state_size.h), np.float32)\n",
    "            self.state_init = [c_init, h_init]\n",
    "            c_in = tf.placeholder(tf.float32, [1, lstm_cell.state_size.c])\n",
    "            h_in = tf.placeholder(tf.float32, [1, lstm_cell.state_size.h])\n",
    "            self.state_in = (c_in, h_in)\n",
    "            rnn_in = tf.expand_dims(hidden, [0])\n",
    "            step_size = tf.shape(self.prev_rewards)[:1]\n",
    "            state_in = tf.contrib.rnn.LSTMStateTuple(c_in, h_in)\n",
    "            lstm_outputs, lstm_state = tf.nn.dynamic_rnn(\n",
    "                lstm_cell, rnn_in, initial_state=state_in, sequence_length=step_size,\n",
    "                time_major=False)\n",
    "            lstm_c, lstm_h = lstm_state\n",
    "            self.state_out = (lstm_c[:1, :], lstm_h[:1, :])\n",
    "            rnn_out = tf.reshape(lstm_outputs, [-1, 48])\n",
    "            \n",
    "            self.actions = tf.placeholder(shape=[None],dtype=tf.int32)\n",
    "            self.actions_onehot = tf.one_hot(self.actions,a_size,dtype=tf.float32)\n",
    "                        \n",
    "            #Output layers for policy and value estimations\n",
    "            self.policy = slim.fully_connected(rnn_out,a_size,\n",
    "                activation_fn=tf.nn.softmax,\n",
    "                weights_initializer=normalized_columns_initializer(0.01),\n",
    "                biases_initializer=None)\n",
    "            self.value = slim.fully_connected(rnn_out,1,\n",
{'fraction_non_alphanumeric': 0.17276043777867855, 'fraction_numerical': 0.014268342115930279, 'mean_word_length': 2.0489372219476025, 'pattern_counts': {'":': 56, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '770247', 'n_tokens_mistral': 4520, 'n_tokens_neox': 4399, 'n_words': 675}