<commit_before>import ppaquette_gym_doom
import ppaquette_gym_doom.wrappers
import numpy.random as npr
import numpy as np
import gym
import tensorflow as tf
from PIL import Image

ENV = gym.make('ppaquette/DoomBasic-v0')
wrapper = ppaquette_gym_doom.wrappers.action_space.ToDiscrete('minimal')
ENV = wrapper(ENV)
X, Y = 320, 240
#X, Y = 160, 120
XY = X * Y
wrapper = ppaquette_gym_doom.wrappers.observation_space.SetResolution('%sx%s'%(X, Y))
ENV = wrapper(ENV)
#wrapper = ppaquette_gym_doom.wrappers.control.SetPlayingMode('human')
#ENV = wrapper(ENV)

def conv2d(x, W):
  return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')

def max_pool_2x2(x):
  return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')

class DoomAgent(object):
    def __init__(self):
        self.episode_states_rgb = tf.placeholder(tf.float32, [None, Y, X, 3])
        self.episode_states_crop = self.episode_states_rgb[:, 95:115, :, 0:1]
        crop = self.episode_states_crop
        self.thr_crop = tf.select(crop < 200, 255*tf.ones_like(crop), tf.zeros_like(crop))
        batch_size = tf.shape(self.episode_states_rgb)[0]

        #first convolution layer
        W_conv1 = tf.Variable(tf.truncated_normal([5, 5, 1, 8], stddev = 0.1), "Wconv1")
        b_conv1 = tf.Variable(tf.zeros([8]), "bconv1")

        h_conv1 = tf.nn.relu(conv2d(crop, W_conv1) + b_conv1)
        h_pool1 = max_pool_2x2(h_conv1)

        #second convolution layer
        W_conv2 = tf.Variable(tf.truncated_normal([5, 5, 8, 8], stddev = 0.1), "Wconv2")
        b_conv2 = tf.Variable(tf.zeros([8]), "bconv2")

        h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
        h_pool2 = max_pool_2x2(h_conv2)
        self.h_pool2 = h_pool2
        flat_size = 5 * 80 * 8 
        h_pool2_flat = tf.reshape(h_pool2, [-1, flat_size])


        #one fully connected layer so far
        W1 = tf.Variable(tf.zeros([flat_size, 32]))
        b1 = tf.Variable(tf.zeros([32]))
        O1 = tf.nn.relu(tf.matmul(h_pool2_flat, W1) + b1)

        #output layer
        W2 = tf.Variable(tf.zeros([32, 3]))
        b2 = tf.Variable(tf.zeros([1, 3]))
        self.O2 = tf.nn.softmax(tf.matmul(O1, W2) + b2)

        #state-value layer
        W3 = tf.Variable(tf.zeros([32, 1]))
        b3 = tf.Variable(tf.zeros([1]))
        self.V = tf.matmul(O1, W3) + b3

        self.actions = tf.placeholder(tf.uint8, [None], 'actions')
        self.actions_one_hot = tf.one_hot(self.actions, 3, on_value = 1.0, off_value = 0.0, axis = -1)
        self.rewards = tf.placeholder(tf.float32, [None, 1], 'rewards')

        self.per_action_log_prob = tf.reduce_sum(self.O2 * self.actions_one_hot, reduction_indices = [1])
        self.entropy = - tf.reduce_mean(tf.log(self.per_action_log_prob) * self.per_action_log_prob)
        self.loss = -tf.reduce_sum((self.rewards - self.V) * tf.log(self.per_action_log_prob)) - 0.01 * self.entropy
        self.train_step = tf.train.AdamOptimizer().minimize(self.loss)
        self.value_loss = tf.reduce_mean(tf.square(self.rewards - self.V))
        self.v_train_step = tf.train.AdamOptimizer().minimize(self.value_loss)

        self.saver = tf.train.Saver([W2, b2, W1, b1, W_conv1, W_conv2, b_conv1, b_conv2, W3, b3])

        init = tf.initialize_all_variables()
        self.sess = tf.Session()
        self.sess.run(init)

    def start_episode(self):
        self.episode_rewards = []
        self.episode_observations = []
        self.episode_actions = []

    def feedback(self, reward):
        self.episode_rewards.append(reward)

    def end_episode(self):
        rewards = np.array(self.episode_rewards)
        rewards = np.cumsum(rewards[::-1])[::-1]
        rewards = rewards.reshape((rewards.shape[0], 1))
        observations = np.array(self.episode_observations)
        actions = np.array(self.episode_actions)

        _t1, _t2, v, loss, vloss, entropy = self.sess.run([self.train_step, self.v_train_step, self.V, self.loss, self.value_loss, self.entropy], feed_dict={self.episode_states_rgb: observations, self.rewards: rewards, self.actions: actions})
        #loss, o2, t = self.sess.run([self.loss, self.O2, self.per_action_log_prob], feed_dict={self.episode_states_rgb: observations, self.rewards: rewards, self.actions: actions})
        print 'loss', loss, 'value loss', vloss, v.shape, entropy
        #print 'o2', o2 
        #print 'b2', b2

    def act(self, ob):
        ob_as_batch = ob.reshape((1, Y, X, 3))
        [o2] = self.sess.run([self.O2], feed_dict={self.episode_states_rgb: ob_as_batch})
        #[h2] = self.sess.run([self.h_pool2], feed_dict={self.episode_states_rgb: ob_as_batch})
        #[crop, pool] = self.sess.run([self.thr_crop, self.h_pool2], feed_dict={self.episode_states_rgb: ob_as_batch})
        #print pool.shape
        #exit(0)
        #grey_image = grey_image.reshape((Y, X))
        #print grey_image.shape
        #crop = crop[0, :, :, 0]
        #crop[crop < 200] = 0
        #print crop.shape, pool.shape
        #img = Image.fromarray(crop, "F")
        #img.show()
        #exit(0)
        #save('my.png')
        #print h2
        #print h2.shape
        #exit(0)
        action = npr.multinomial(1, o2[0, :]).argmax() 

        return action + 1 #no NOOP
    def record_action(self, action, ob):
        self.episode_actions.append(action - 1)
        self.episode_observations.append(ob)

def learn(episodes, max_steps_per_episode, visualisation_freq):
    outdir = './output/'
    agent = DoomAgent()

    for episode in xrange(episodes):
        if (episode + 1) % visualisation_freq == 0:
            ENV.monitor.start(outdir, force=True)
        done = False
        ob = ENV.reset()
        rewards = []
        agent.start_episode()
        j = 0
        last_action = 0
        while not done and j < 5 * 35: #3 sec
            if j % 5 == 0:
                action = agent.act(ob)
                last_action = action
                agent.record_action(action, ob)
                new_ob, reward, done, info = ENV.step(action)
                agent.feedback(reward)
                if done:
                    break
            else:
                action = last_action
                new_ob, reward, done, info = ENV.step(action)
                if done:
                    agent.record_action(action, ob)
                    agent.feedback(reward)
                    break

            ob = new_ob
            j += 1

        print info['TOTAL_REWARD']
        agent.end_episode()
        if (episode + 1) % visualisation_freq == 0:
            ENV.monitor.close()
            agent.saver.save(agent.sess, 'chckpoints/model', global_step=episode)

if __name__ == '__main__':
    learn(50000, None, 100)<commit_msg>successful BasicDoom learning<commit_after>import ppaquette_gym_doom
from collections import deque
import ppaquette_gym_doom.wrappers
import numpy.random as npr
import numpy as np
import random
import gym
from gym import wrappers
import prettytensor as pt
import tensorflow as tf
from PIL import Image
from scipy.misc import logsumexp
#from skimage.color import rgb2gray
import cv2
from skimage.transform import resize

ENV = gym.make('ppaquette/DoomBasic-v0')
ENV = wrappers.SkipWrapper(3)(ENV)
wrapper = ppaquette_gym_doom.wrappers.action_space.ToDiscrete('minimal')
ENV = wrapper(ENV)
X, Y = 320, 240
wrapper = ppaquette_gym_doom.wrappers.observation_space.SetResolution('%sx%s'%(X, Y))
XY = X * Y
ENV = wrapper(ENV)
crop_X_l, crop_X_u = 95, 120

def conv2d(x, W):
  return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')

def max_pool_2x2(x):
  return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')

class DoomAgent(object):
    def __init__(self, restore = False):
        patch_size = 3
        patch = patch_size
        num_channels = 1
        depth = 4
        num_hidden = 32
	num_labels = 3

        self.episode_states_rgb = tf.placeholder(tf.float32, [None, crop_X_u - crop_X_l, X, num_channels])
        self.policy_net, _ = (pt.wrap(self.episode_states_rgb)
            .conv2d(patch, depth, stride = (2,2), batch_normalize=True)
            .conv2d(patch, depth, stride = (2,2), batch_normalize=True)
            .flatten()
            .dropout(0.5)
            .fully_connected(num_hidden, activation_fn=tf.nn.relu)
            .softmax_classifier(3))

        self.actions = tf.placeholder(tf.uint8, [None], 'actions')
        self.actions_one_hot = tf.one_hot(self.actions, 3, on_value = 1.0, off_value = 0.0, axis = -1)
        self.rewards = tf.placeholder(tf.float32, [None, 1], 'rewards')

        self.per_action_log_prob = tf.reduce_sum(tf.mul(self.policy_net, self.actions_one_hot), reduction_indices = [1])
        self.entropy = tf.reduce_mean(self.policy_net * tf.log(self.policy_net), reduction_indices = [1])
        self.entropy = tf.reduce_mean(self.entropy)
        self.loss = -tf.reduce_mean(self.rewards * tf.log(self.per_action_log_prob)) + 1 * self.entropy

        self.train_step = tf.train.AdamOptimizer(1e-4).minimize(self.loss)
        self.saver = tf.train.Saver(max_to_keep=5)

        init = tf.initialize_all_variables()
        self.sess = tf.Session()
        self.sess.run(init)
        if restore:
            print 'saver restore'
            self.saver.restore(self.sess, tf.train.latest_checkpoint('./chckpoints'))

    def start_episode(self):
        self.episode_rewards = []
        self.episode_observations = []
        self.episode_actions = []

    def feedback(self, reward):
        self.episode_rewards.append(reward)

    def end_episode(self):
        rewards = np.array(self.episode_rewards)
        rewards = (np.cumsum(rewards[::-1])[::-1])  * 0.01
        rewards = rewards.reshape((rewards.shape[0], 1))
        observations = np.array(self.episode_observations)
        actions = np.array(self.episode_actions)

        self.sess.run(self.train_step, feed_dict={self.episode_states_rgb: observations, self.rewards: rewards, self.actions: actions})

    def transform_input(self, ob):
        #obBGR = cv2.cvtColor(ob, cv2.COLOR_RGB2BGR)
        #cv2.imshow('before', obBGR)
        ob = cv2.cvtColor(ob, cv2.COLOR_RGB2GRAY)
        ob = ob[crop_X_l:crop_X_u, :]
        #image = cv2.resize(image[0], (width, height), interpolation=cv2.INTER_AREA)

        #cv2.imshow('after', ob)
        #raw_input()
        #exit(0)
        return ob.reshape((ob.shape[0], ob.shape[1], 1))

    def act(self, ob, dump = False):
        ob = self.transform_input(ob)
        ob_as_batch = ob.reshape((1, ob.shape[0], ob.shape[1], ob.shape[2]))
        [softmax] = self.sess.run([self.policy_net], feed_dict={self.episode_states_rgb: ob_as_batch})

        softmax = softmax[0, :]
        if dump:
            print softmax
        try:
            action = npr.multinomial(1, softmax).argmax()
        except:
            action = softmax.argmax()

        return action + 1 #no NOOP
    def record_action(self, action, ob):
        self.episode_actions.append(action - 1)
        ob = self.transform_input(ob)
        self.episode_observations.append(ob)

def learn(episodes, visualisation_freq, restore, monitor):
    outdir = './output/'
    agent = DoomAgent(restore)
    env = ENV
    if monitor:
        env = wrappers.Monitor(ENV, './output/', force=True)
    for episode in xrange(episodes):
        done = False
        ob = env.reset()
        agent.start_episode()
        j = 0
        last_action = 0
        while not done:
            dump = (episode + 1) % visualisation_freq == 0
            action = agent.act(ob, dump)
            last_action = action
            agent.record_action(action, ob)
            new_ob, reward, done, info = env.step(action)
            agent.feedback(reward)
            if done:
                break
            ob = new_ob
            j += 1

        print info['TOTAL_REWARD']
        agent.end_episode()
        if (episode + 1) % visualisation_freq == 0:
            agent.saver.save(agent.sess, 'chckpoints/model', global_step=episode)

if __name__ == '__main__':
    import argparse
    import sys
    parser = argparse.ArgumentParser()
    parser.add_argument('--restore', type=bool, default=False)
    parser.add_argument('--monitor', type=bool, default=False)
    parser.add_argument('--dump_freq', type=int, default=100)
    args = parser.parse_args()

    print args
    learn(50000, args.dump_freq, args.restore, args.monitor)
{'fraction_non_alphanumeric': 0.132043530834341, 'fraction_numerical': 0.025312374042724708, 'mean_word_length': 2.63386057410662, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '25404', 'n_tokens_mistral': 4657, 'n_tokens_neox': 4386, 'n_words': 849}