# Path: baselines/deepq/prediction/tfacvp/tf_ops.py
# def ReLu(x, name, reuse=False):
#     with tf.variable_scope(name, reuse=reuse) as scope:
#         l = tf.nn.relu(x)
#     return l
# 
# def Conv2D(x, filter_shape, out_dim, strides, padding, name, reuse=False):
#     # x: input tensor (float32)[n, w, h, c]
#     # filter_shape: conv2d filter (int)[w, h]
#     # out_dim: output channels (int)
#     # strides: conv2d stride size (int)
#     # padding: padding type (str)
#     # name: variable scope (str)
#            
#     with tf.variable_scope(name, reuse=reuse) as scope:
#         in_dim = x.get_shape()[-1]
#         w = tf.get_variable('w', shape=filter_shape + [in_dim, out_dim], initializer=tf.contrib.layers.xavier_initializer_conv2d(uniform=True))
#         b = tf.get_variable('b', shape=[out_dim], initializer=tf.constant_initializer(0.0))
#         l = tf.nn.conv2d(x, w, strides=[1, strides, strides, 1], padding=padding, name='conv2d')
#         l = tf.nn.bias_add(l, b, name='bias_add')
#     return l
# 
# def FC(x, out_dim, name, initializer=tf.contrib.layers.xavier_initializer(uniform=True), reuse=False):
#     # x: input tensor (float32)[n, in_dim]
#     # out_dim: output channels (int)
#     # name: variable scope (str)
# 
#     x = tf.contrib.layers.flatten(x)
#     with tf.variable_scope(name, reuse=reuse) as scope:
#         in_dim = x.get_shape()[-1]
#         w = tf.get_variable('w', shape=[in_dim, out_dim], initializer=initializer)
#         b = tf.get_variable('b', shape=[out_dim], initializer=tf.constant_initializer(0.0))
#         l = tf.add(tf.matmul(x, w), b, name='add')
#     return l
# 
# def Deconv2D(x, filter_shape, output_shape, out_dim, strides, padding, name, reuse=False):
#     # x: input tensor (float32) [n, w, h, c]
#     # filter_shape: conv2d filter (int)[w, h]
#     # out_dim: output channels (int)
#     # strides: conv2d stride size (int)
#     # padding: padding type (str)
#     # name: variable scope (str)
# 
#     with tf.variable_scope(name, reuse=reuse) as scope:
#         in_dim = x.get_shape()[-1]
#         w = tf.get_variable('w', shape=filter_shape + [out_dim, in_dim], initializer=tf.contrib.layers.xavier_initializer_conv2d(uniform=True))
#         b = tf.get_variable('b', shape=[out_dim], initializer=tf.constant_initializer(0.0))
#         l = tf.nn.conv2d_transpose(x, w, output_shape=output_shape, strides=[1, strides, strides, 1], padding=padding, name='deconv2d')
#         l = tf.nn.bias_add(l, b, name='bias_add')
#     return l


import tensorflow as tf
import numpy as np
import re
from .tf_ops import ReLu, Conv2D, FC, Deconv2D
        with tf.variable_scope('optimize', reuse=not self.is_train) as scope:
            # Setup global_step, optimizer
            self.global_step = tf.get_variable('global_step', shape=(), initializer=tf.constant_initializer(0.0), trainable=False)

            self.learning_rate = tf.train.exponential_decay(lr, self.global_step, 1e5, 0.9, staircase=True)
            self.optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate, name='optimizer')

            # According to original paper code, learning rate of bias is 2x of base learning rate
            grads_vars = self.optimizer.compute_gradients(self.loss)
            bias_pattern = re.compile('.*/b')
            grads_vars_mult = []
            for grad, var in grads_vars:
                if bias_pattern.match(var.op.name):
                    grads_vars_mult.append((grad * 2.0, var))
                else:
                    grads_vars_mult.append((grad, var))

            # According to original paper, gradient should be clipped with [-0.1, 0.1]
            grads_clip = [(tf.clip_by_value(grad, -0.1, 0.1), var) for grad, var in grads_vars_mult]
            self.train = self.optimizer.apply_gradients(grads_clip, global_step=self.global_step)

    def _create_encoder(self, x):
        # x: input image (tensor([batch_size, 84, 84, 12]))
        l = Conv2D(x, [6, 6], 64, 2, 'VALID', 'conv1')
        l = ReLu(l, 'relu1')
        l = Conv2D(l, [6, 6], 64, 2, 'SAME', 'conv2')
        l = ReLu(l, 'relu2')
        l = Conv2D(l, [6, 6], 64, 2, 'SAME', 'conv3')
        l = ReLu(l, 'relu3')
        l = FC(l, 1024, 'ip1')

# Path: baselines/deepq/prediction/tfacvp/tf_ops.py
# def ReLu(x, name, reuse=False):
#     with tf.variable_scope(name, reuse=reuse) as scope:
#         l = tf.nn.relu(x)
#     return l
# 
# def Conv2D(x, filter_shape, out_dim, strides, padding, name, reuse=False):
#     # x: input tensor (float32)[n, w, h, c]
#     # filter_shape: conv2d filter (int)[w, h]
#     # out_dim: output channels (int)
#     # strides: conv2d stride size (int)
#     # padding: padding type (str)
#     # name: variable scope (str)
#            
#     with tf.variable_scope(name, reuse=reuse) as scope:
#         in_dim = x.get_shape()[-1]
#         w = tf.get_variable('w', shape=filter_shape + [in_dim, out_dim], initializer=tf.contrib.layers.xavier_initializer_conv2d(uniform=True))
#         b = tf.get_variable('b', shape=[out_dim], initializer=tf.constant_initializer(0.0))
#         l = tf.nn.conv2d(x, w, strides=[1, strides, strides, 1], padding=padding, name='conv2d')
#         l = tf.nn.bias_add(l, b, name='bias_add')
#     return l
# 
# def FC(x, out_dim, name, initializer=tf.contrib.layers.xavier_initializer(uniform=True), reuse=False):
#     # x: input tensor (float32)[n, in_dim]
#     # out_dim: output channels (int)
#     # name: variable scope (str)
# 
#     x = tf.contrib.layers.flatten(x)
#     with tf.variable_scope(name, reuse=reuse) as scope:
#         in_dim = x.get_shape()[-1]
#         w = tf.get_variable('w', shape=[in_dim, out_dim], initializer=initializer)
#         b = tf.get_variable('b', shape=[out_dim], initializer=tf.constant_initializer(0.0))
#         l = tf.add(tf.matmul(x, w), b, name='add')
#     return l
# 
# def Deconv2D(x, filter_shape, output_shape, out_dim, strides, padding, name, reuse=False):
#     # x: input tensor (float32) [n, w, h, c]
#     # filter_shape: conv2d filter (int)[w, h]
#     # out_dim: output channels (int)
#     # strides: conv2d stride size (int)
#     # padding: padding type (str)
#     # name: variable scope (str)
# 
#     with tf.variable_scope(name, reuse=reuse) as scope:
#         in_dim = x.get_shape()[-1]
#         w = tf.get_variable('w', shape=filter_shape + [out_dim, in_dim], initializer=tf.contrib.layers.xavier_initializer_conv2d(uniform=True))
#         b = tf.get_variable('b', shape=[out_dim], initializer=tf.constant_initializer(0.0))
#         l = tf.nn.conv2d_transpose(x, w, output_shape=output_shape, strides=[1, strides, strides, 1], padding=padding, name='deconv2d')
#         l = tf.nn.bias_add(l, b, name='bias_add')
#     return l


# Path: baselines/deepq/prediction/tfacvp/model.py
import tensorflow as tf
import numpy as np
import re
from .tf_ops import ReLu, Conv2D, FC, Deconv2D

        with tf.variable_scope('optimize', reuse=not self.is_train) as scope:
            # Setup global_step, optimizer
            self.global_step = tf.get_variable('global_step', shape=(), initializer=tf.constant_initializer(0.0), trainable=False)

            self.learning_rate = tf.train.exponential_decay(lr, self.global_step, 1e5, 0.9, staircase=True)
            self.optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate, name='optimizer')

            # According to original paper code, learning rate of bias is 2x of base learning rate
            grads_vars = self.optimizer.compute_gradients(self.loss)
            bias_pattern = re.compile('.*/b')
            grads_vars_mult = []
            for grad, var in grads_vars:
                if bias_pattern.match(var.op.name):
                    grads_vars_mult.append((grad * 2.0, var))
                else:
                    grads_vars_mult.append((grad, var))

            # According to original paper, gradient should be clipped with [-0.1, 0.1]
            grads_clip = [(tf.clip_by_value(grad, -0.1, 0.1), var) for grad, var in grads_vars_mult]
            self.train = self.optimizer.apply_gradients(grads_clip, global_step=self.global_step)

    def _create_encoder(self, x):
        # x: input image (tensor([batch_size, 84, 84, 12]))
        l = Conv2D(x, [6, 6], 64, 2, 'VALID', 'conv1')
        l = ReLu(l, 'relu1')
        l = Conv2D(l, [6, 6], 64, 2, 'SAME', 'conv2')
        l = ReLu(l, 'relu2')
        l = Conv2D(l, [6, 6], 64, 2, 'SAME', 'conv3')
        l = ReLu(l, 'relu3')
        l = FC(l, 1024, 'ip1')

        l = ReLu(l, 'relu4')