<commit_before>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reward function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import puzzle\n",
    "\n",
    "def reward(self):\n",
    "    increment = self.game.increment\n",
    "    totalScore = self.game.totalScore\n",
    "    # loose does not work yet\n",
    "    #loose = self.game.loose\n",
    "    return increment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Network Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.optimizers import RMSprop\n",
    "\n",
    "class DQNAgent:\n",
    "    def __init__(self, state_size, action_size):\n",
    "        self.state_size = state_size\n",
    "        self.action_size = action_size\n",
    "        self.memory = deque(maxlen=100000)\n",
    "        self.gamma = 0.9    # discount rate\n",
    "        self.epsilon = 1.0  # exploration rate\n",
    "        self.e_decay = .99\n",
    "        self.e_min = 0.05\n",
    "        self.learning_rate = 0.01\n",
    "        self.model = self._build_model()\n",
    "\n",
    "    def _build_model(self):\n",
    "        # Neural Net for Deep-Q learning Model\n",
    "        model = Sequential()\n",
    "        model.add(Dense(20, input_dim=self.state_size, activation='tanh')) # Adds the first layer with 16 inputs\n",
    "        model.add(Dense(20, activation='tanh', init='uniform')) # Adds Hidden layer with 20 nodes\n",
    "        model.add(Dense(self.action_size, activation='linear')) # Adds output layer with 2 nodes\n",
    "        model.compile(loss='mse',\n",
    "                      optimizer=RMSprop(lr=self.learning_rate)) # Creates the model from all of the above\n",
    "        return model\n",
    "\n",
    "    def remember(self, state, action, reward, next_state, done):\n",
    "        self.memory.append((state, action, reward, next_state, done))\n",
    "\n",
    "    def act(self, state):\n",
    "        if np.random.rand() <= self.epsilon:\n",
    "            return random.randrange(self.action_size)\n",
    "        act_values = self.model.predict(state)\n",
    "        return np.argmax(act_values[0])  # returns action\n",
    "\n",
    "    def replay(self, batch_size):\n",
    "        batch_size = min(batch_size, len(self.memory))\n",
    "        minibatch = random.sample(self.memory, batch_size)\n",
    "        X = np.zeros((batch_size, self.state_size))\n",
    "        Y = np.zeros((batch_size, self.action_size))\n",
    "        for i in range(batch_size):\n",
    "            state, action, reward, next_state, done = minibatch[i]\n",
    "            target = self.model.predict(state)[0]\n",
    "            if done:\n",
    "                target[action] = reward\n",
    "            else:\n",
    "                target[action] = reward + self.gamma * \\\n",
    "                            np.amax(self.model.predict(next_state)[0])\n",
    "            X[i], Y[i] = state, target\n",
    "        self.model.fit(X, Y, batch_size=batch_size, nb_epoch=1, verbose=0)\n",
    "        if self.epsilon > self.e_min:\n",
    "            self.epsilon *= self.e_decay\n",
    "\n",
    "    def load(self, name):\n",
    "        self.model.load_weights(name)\n",
    "\n",
    "    def save(self, name):\n",
    "        self.model.save_weights(name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Main Game execution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "4\n",
      "4\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "4\n",
      "4\n",
      "0\n",
      "0\n",
      "4\n",
      "4\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "4\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "4\n",
      "4\n",
      "0\n",
      "4\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "4\n",
      "0\n",
      "0\n",
      "4\n",
      "4\n",
      "8\n",
      "0\n",
      "4\n",
      "4\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "4\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "4\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "# %load machine.py\n",
    "import time\n",
    "import puzzle\n",
    "\n",
    "\n",
    "EPISODES = 10\n",
    "\n",
    "\n",
    "#ACCESS SCORE AS self.game.score\n",
    "#ACCESS MATRIX AS self.game.matrix\n",
    "#DECIDE ACTION TO TAKE IN act()\n",
    "#POSSIBLE ACTIONS:\n",
    "#\tgo up:\t\t\"'w'\"\n",
    "#\tgo left:\t\"'a'\"\n",
    "#\tgo right:\t\"'s'\"\n",
    "#\tgo down:\t\"'d'\"\n",
    "\n",
    "class Machine:\n",
    "    game=puzzle.GameGrid() #Game object\n",
    "    def run(self):\n",
    "        for i in range(EPISODES):\n",
    "            \n",
    "            self.game.restart()\n",
    "            for t in range(20):\n",
    "                #self.game.key_down(\"'w'\") #EXAMPLE UNCOMMENT TO RUN\n",
    "                #print(self.game.score)\n",
    "                #print(self.game.matrix)\n",
    "                print(self.reward())\n",
    "                self.game.key_down(self.act('up')) #COMMENT TO RUN EXAMPLE\n",
    "                self.game.update_idletasks\n",
    "                self.game.update()\n",
    "                time.sleep(0.1)\n",
    "            \n",
    "        #print(\"episode: {}/{}, score: {}, e: {:.2}\".format(e, EPISODES, time, agent.epsilon))\n",
    "\n",
    "    def act(self, direction):\n",
    "        up = \"'w'\"\n",
    "        down = \"'s'\"\n",
    "        right = \"'d'\"\n",
    "        left = \"'a'\"\n",
    "        if direction == 'up': return up\n",
    "        elif direction == 'down': return down\n",
    "        elif direction == 'right': return right\n",
    "        elif direction == 'left': return left\n",
    "        \n",
    "    def reward(self):\n",
    "        increment = self.game.increment\n",
    "        #totalScore = self.game.score\n",
    "        loose = self.game.result\n",
    "        l = 0\n",
    "        if loose: l = -2*increment\n",
    "        #print(\"loose: \" + str(loose))\n",
    "        return (increment + l)\n",
    "    \n",
    "    #######################################\n",
    "    # I made this function to calculate the best next step. This will be the function\n",
    "    # Determening which state to be given to the neural network.\n",
    "    # This is a bad implementation since it relies on copying the whole game for every\n",
    "    # calculation, but it will work to train in the beginning. \n",
    "    def calculateNextStep(self, game):\n",
    "        best_qval = 0\n",
    "        best_move = 'up' # Default move\n",
    "        acts = ['up','down','right','left']\n",
    "        for act in acts: \n",
    "            game_copy = game\n",
    "            game_copy.key_down(self.act(act))\n",
    "            self.game.update_idletasks\n",
    "            self.game.update()\n",
    "            qval = game_copy.reward()\n",
    "            if qval > best_qval:\n",
    "                best_qval = qval\n",
    "                best_move = act\n",
    "        return best_move\n",
    "    #######################################\n",
    "        \n",
    "our_machine = Machine()\n",
    "our_machine.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[2, 4, 2, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]\n",
      "Epoch 1/10\n",
      "4/4 [==============================] - 0s - loss: 0.4919\n",
      "Epoch 2/10\n",
      "4/4 [==============================] - 0s - loss: 0.2883\n",
      "Epoch 3/10\n",
      "4/4 [==============================] - 0s - loss: 0.2134\n",
      "Epoch 4/10\n",
      "4/4 [==============================] - 0s - loss: 0.1772\n",
      "Epoch 5/10\n",
      "4/4 [==============================] - 0s - loss: 0.1708\n",
      "Epoch 6/10\n",
      "4/4 [==============================] - 0s - loss: 0.1691\n",
      "Epoch 7/10\n",
      "4/4 [==============================] - 0s - loss: 0.1693\n",
      "Epoch 8/10\n",
      "4/4 [==============================] - 0s - loss: 0.1706\n",
      "Epoch 9/10\n",
      "4/4 [==============================] - 0s - loss: 0.1724\n",
      "Epoch 10/10\n",
      "4/4 [==============================] - 0s - loss: 0.1749\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'action' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-34528039ea0c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     41\u001b[0m \u001b[0;31m# So that the neural network don't \"forget\" the old state with new experiences\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m \u001b[0;31m# we make a function to remember. (Already implemented perfectly in DQNAgent)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 43\u001b[0;31m \u001b[0mmemory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnext_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mremember\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnext_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'action' is not defined"
     ]
    }
   ],
   "source": [
    "# Easier implemtation to start with. I have used this guide: https://keon.io/deep-q-learning/\n",
    "# The following code is to see that the neural network and deep q network is setup correctly\n",
    "\n",
    "# 1\n",
    "############## Simple Neural Net #################\n",
    "# Here are some more concrete numbers to put into the neural network than in the DQNAgent\n",
    "model = Sequential()\n",
    "model.add(Dense(20, input_dim=4, activation='tanh')) # Adds the first layer with 16 inputs\n",
    "model.add(Dense(20, activation='tanh'))              # Adds Hidden layer with 20 nodes (Removed 'uniform')\n",
    "model.add(Dense(4, activation='linear'))             # Adds output layer with 2 nodes\n",
    "model.compile(loss='mse',optimizer=RMSprop(lr=0.01)) # Creates the model from all of the above\n",
    "\n",
    "#     This training process makes the neural net predict \n",
    "#     the reward value (target_f) from a certain state.\n",
    "\n",
    "state = our_machine.game.matrix  # Game state matrix ( Converting to a 1D array to please model.fit())                      \n",
    "print(state)\n",
    "\n",
    "\n",
    "# After some debugging the target has to be a table just like the state\n",
    "# Therefore I am wondering if we should have the target as the complete theoretical board?\n",
    "# Or as the best possible next state? \n",
    "\n",
    "# Theoretical limit target_f = [[131072, 65536, 32768, 16384], [1024, 2048, 4096, 8192], [512, 256, 128, 64], [4, 8, 16, 32]]\n",
    "# I we want to use the best possible next state we first have to: \n",
    "    # 1: Calculate the next four possibilities\n",
    "    # 2: Pick the one with highest q-value\n",
    "    # 3: Use that next state as target_f\n",
    "    \n",
    "# For now I will use this random state\n",
    "target_f = [[0, 0, 2, 0], [0, 0, 0, 0], [0, 0, 2, 0], [0, 0, 0, 0]]\n",
    "\n",
    "model.fit(state, target_f)        # Fitting the data (Feeding the network)\n",
    "prediction = model.predict(state) # Predicting the reward from the current state\n",
    "\n",
    "\n",
    "\n",
    "# 2\n",
    "############## Deep Q Network (DQN) #################\n",
    "\n",
    "# So that the neural network don't \"forget\" the old state with new experiences\n",
    "# we make a function to remember. (Already implemented perfectly in DQNAgent)\n",
    "memory = ((state, action, reward, next_state, done))\n",
    "\n",
    "def remember(self, state, action, reward, next_state, done):\n",
    "    self.memory.append((state, action, reward, next_state, done))\n",
    "\n",
    "# Replay trains the neural network by replaying the previous experiences in its memory\n",
    "# These memories are experienced in batches\n",
    "\n",
    "batch_size = 5 # Just a random number \n",
    "batches = min(batch_size, len(self.memory))\n",
    "batches = np.random.choice(len(self.memory), batches) # Picks random experiences to make batches of\n",
    "\n",
    "\n",
    "# Training\n",
    "    \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}<commit_msg>Created a working copy in the referenceAgent notebook<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reward function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import puzzle\n",
    "\n",
    "def reward(self):\n",
    "    increment = self.game.increment\n",
    "    totalScore = self.game.totalScore\n",
    "    # loose does not work yet\n",
    "    #loose = self.game.loose\n",
    "    return increment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Network Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.optimizers import RMSprop\n",
    "\n",
    "class DQNAgent:\n",
    "    def __init__(self, state_size, action_size):\n",
    "        self.state_size = state_size\n",
    "        self.action_size = action_size\n",
    "        self.memory = deque(maxlen=100000)\n",
    "        self.gamma = 0.9    # discount rate\n",
    "        self.epsilon = 1.0  # exploration rate\n",
    "        self.e_decay = .99\n",
    "        self.e_min = 0.05\n",
    "        self.learning_rate = 0.01\n",
    "        self.model = self._build_model()\n",
    "\n",
    "    def _build_model(self):\n",
    "        # Neural Net for Deep-Q learning Model\n",
    "        model = Sequential()\n",
    "        model.add(Dense(20, input_dim=self.state_size, activation='tanh')) # Adds the first layer with 16 inputs\n",
    "        model.add(Dense(20, activation='tanh', init='uniform')) # Adds Hidden layer with 20 nodes\n",
    "        model.add(Dense(self.action_size, activation='linear')) # Adds output layer with 2 nodes\n",
    "        model.compile(loss='mse',\n",
    "                      optimizer=RMSprop(lr=self.learning_rate)) # Creates the model from all of the above\n",
    "        return model\n",
    "\n",
    "    def remember(self, state, action, reward, next_state, done):\n",
    "        self.memory.append((state, action, reward, next_state, done))\n",
    "\n",
    "    def act(self, state):\n",
    "        if np.random.rand() <= self.epsilon:\n",
    "            return random.randrange(self.action_size)\n",
    "        act_values = self.model.predict(state)\n",
    "        return np.argmax(act_values[0])  # returns action\n",
    "\n",
    "    def replay(self, batch_size):\n",
    "        batch_size = min(batch_size, len(self.memory))\n",
    "        minibatch = random.sample(self.memory, batch_size)\n",
    "        X = np.zeros((batch_size, self.state_size))\n",
    "        Y = np.zeros((batch_size, self.action_size))\n",
    "        for i in range(batch_size):\n",
    "            state, action, reward, next_state, done = minibatch[i]\n",
    "            target = self.model.predict(state)[0]\n",
    "            if done:\n",
    "                target[action] = reward\n",
    "            else:\n",
    "                target[action] = reward + self.gamma * \\\n",
    "                            np.amax(self.model.predict(next_state)[0])\n",
    "            X[i], Y[i] = state, target\n",
    "        self.model.fit(X, Y, batch_size=batch_size, nb_epoch=1, verbose=0)\n",
    "        if self.epsilon > self.e_min:\n",
    "            self.epsilon *= self.e_decay\n",
    "\n",
    "    def load(self, name):\n",
    "        self.model.load_weights(name)\n",
    "\n",
    "    def save(self, name):\n",
    "        self.model.save_weights(name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Main Game execution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import puzzle\n",
    "from collections import deque\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.optimizers import RMSprop\n",
    "import numpy as np\n",
    "\n",
    "EPISODES = 10\n",
    "\n",
    "#ACCESS SCORE AS self.game.score\n",
    "#ACCESS MATRIX AS self.game.matrix\n",
    "#DECIDE ACTION TO TAKE IN act()\n",
    "#POSSIBLE ACTIONS:\n",
    "#\tgo up:\t\t\"'w'\"\n",
    "#\tgo left:\t\"'a'\"\n",
    "#\tgo right:\t\"'s'\"\n",
    "#\tgo down:\t\"'d'\"\n",
    "\n",
    "class Machine:\n",
    "    game=puzzle.GameGrid() # Game object\n",
    "    model = Sequential()\n",
    "    inputVector=np.zeros((1, 16))\n",
    "    Qvalues0=np.zeros((1,4))\n",
    "    Qvalues1=np.zeros((1,4))\n",
    "    acts = [\"'w'\",\"'s'\",\"'d'\",\"'a'\"]\n",
    "    gamma = 0.9    # Discount rate\n",
    "    def __init__(self):\n",
    "        # Create model\n",
    "        self.model.add(Dense(20, input_dim=16, activation='tanh')) # Adds the first layer with 16 inputs\n",
    "        self.model.add(Dense(20, activation='tanh'))              # Adds Hidden layer with 20 nodes (Removed 'uniform')\n",
    "        self.model.add(Dense(4, activation='linear'))             # Adds output layer with 20 nodes\n",
    "        self.model.compile(loss='mse',optimizer=RMSprop(lr=0.01)) # Creates the model from all of the above\n",
    "    def run(self):\n",
    "        # Transform game state to 1D array\n",
    "        for i in range(4):\n",
    "            self.inputVector[0][0+4*i:4+4*i]=self.game.matrix[i]\n",
    "        print(self.inputVector)\n",
    "        self.game.increment=self.reward() # Update reward if game has been lost\n",
    "        self.game.key_down(self.act()) # Select action and update weights\n",
    "        # Game loop\n",
    "        self.game.update_idletasks\n",
    "        self.game.update()\n",
    "        # pool emaG\n",
    "    def act(self):\n",
    "        print(\"ACT\")\n",
    "        # Predict Q values of current state\n",
    "        self.Qvalues1[0]=self.gamma*self.model.predict(self.inputVector)+self.game.increment\n",
    "        # Update weights with respect to last step's prediction of this step's Q values\n",
    "        self.model.train_on_batch(self.inputVector, self.Qvalues0)\n",
    "        # Make this step's Q values next step's past Q values\n",
    "        self.Qvalues0=self.Qvalues1\n",
    "        # Select action with highest Q value\n",
    "        print(\"END ACT\")\n",
    "        return self.acts[self.Qvalues1.argmax()]\n",
    "        \n",
    "    def reward(self):\n",
    "        l=0\n",
    "        if self.game.result: l = -2*self.game.increment-self.game.score/4\n",
    "        return (self.game.increment + l)\n",
    "    \n",
    "    #######################################\n",
    "    # I made this function to calculate the best next step. This will be the function\n",
    "    # Determening which state to be given to the neural network.\n",
    "    # This is a bad implementation since it relies on copying the whole game for every\n",
    "    # calculation, but it will work to train in the beginning. \n",
    "    def calculateNextStep(self, game):\n",
    "        best_qval = 0\n",
    "        best_move = 'up' # Default move\n",
    "        acts = ['up','down','right','left']\n",
    "        for act in acts: \n",
    "            game_copy = game\n",
    "            game_copy.key_down(self.act(act))\n",
    "            self.game.update_idletasks\n",
    "            self.game.update()\n",
    "            qval = game_copy.reward()\n",
    "            if qval > best_qval:\n",
    "                best_qval = qval\n",
    "                best_move = act\n",
    "        return best_move\n",
    "    #######################################\n",
    "        \n",
    "our_machine = Machine()\n",
    "while True:\n",
    "    our_machine.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Easier implemtation to start with. I have used this guide: https://keon.io/deep-q-learning/\n",
    "# The following code is to see that the neural network and deep q network is setup correctly\n",
    "\n",
    "# 1\n",
    "############## Simple Neural Net #################\n",
    "# Here are some more concrete numbers to put into the neural network than in the DQNAgent\n",
    "model = Sequential()\n",
    "model.add(Dense(20, input_dim=4, activation='tanh')) # Adds the first layer with 16 inputs\n",
    "model.add(Dense(20, activation='tanh'))              # Adds Hidden layer with 20 nodes (Removed 'uniform')\n",
    "model.add(Dense(4, activation='linear'))             # Adds output layer with 2 nodes\n",
    "model.compile(loss='mse',optimizer=RMSprop(lr=0.01)) # Creates the model from all of the above\n",
    "\n",
    "#     This training process makes the neural net predict \n",
    "#     the reward value (target_f) from a certain state.\n",
    "\n",
    "state = our_machine.game.matrix  # Game state matrix ( Converting to a 1D array to please model.fit())                      \n",
    "print(state)\n",
    "\n",
    "\n",
    "# After some debugging the target has to be a table just like the state\n",
    "# Therefore I am wondering if we should have the target as the complete theoretical board?\n",
    "# Or as the best possible next state? \n",
    "\n",
    "# Theoretical limit target_f = [[131072, 65536, 32768, 16384], [1024, 2048, 4096, 8192], [512, 256, 128, 64], [4, 8, 16, 32]]\n",
    "# I we want to use the best possible next state we first have to: \n",
    "    # 1: Calculate the next four possibilities\n",
    "    # 2: Pick the one with highest q-value\n",
    "    # 3: Use that next state as target_f\n",
    "    \n",
    "# For now I will use this random state\n",
    "target_f = [[0, 0, 2, 0], [0, 0, 0, 0], [0, 0, 2, 0], [0, 0, 0, 0]]\n",
    "\n",
    "model.fit(state, target_f)        # Fitting the data (Feeding the network)\n",
    "prediction = model.predict(state) # Predicting the reward from the current state\n",
    "\n",
    "\n",
    "\n",
    "# 2\n",
    "############## Deep Q Network (DQN) #################\n",
    "\n",
    "# So that the neural network don't \"forget\" the old state with new experiences\n",
    "# we make a function to remember. (Already implemented perfectly in DQNAgent)\n",
    "memory = ((state, action, reward, next_state, done))\n",
    "\n",
    "def remember(self, state, action, reward, next_state, done):\n",
    "    self.memory.append((state, action, reward, next_state, done))\n",
    "\n",
    "# Replay trains the neural network by replaying the previous experiences in its memory\n",
    "# These memories are experienced in batches\n",
    "\n",
    "batch_size = 5 # Just a random number \n",
    "batches = min(batch_size, len(self.memory))\n",
    "batches = np.random.choice(len(self.memory), batches) # Picks random experiences to make batches of\n",
    "\n",
    "\n",
    "# Training\n",
    "    \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "a=np.array([[1,2,7],[3,4,5]])\n",
    "b=np.array([0,0,0,0,0,0])\n",
    "for i in range(2):\n",
    "    b[0+3*i:3+3*i]=a[i]\n",
    "print(b[b.argmax()])\n",
    "print(b.shape)\n",
    "x_test = np.zeros((1, 16))\n",
    "print(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
{'fraction_non_alphanumeric': 0.2136542578907383, 'fraction_numerical': 0.041439325920298364, 'mean_word_length': 1.941194393662401, 'pattern_counts': {'":': 138, '<': 7, '<?xml version=': 0, '>': 10, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6506121', 'n_tokens_mistral': 11672, 'n_tokens_neox': 10983, 'n_words': 2254}