# Path: network/Util.py
# def create_batch_norm_vars(n_out, tower_setup, scope_name="bn"):
#   with tf.device(tower_setup.variable_device), tf.variable_scope(scope_name):
#     initializer_zero = tf.constant_initializer(0.0, dtype=tf.float32)
#     beta = tf.get_variable("beta", [n_out], tf.float32, initializer_zero)
#     initializer_gamma = tf.constant_initializer(1.0, dtype=tf.float32)
#     gamma = tf.get_variable("gamma", [n_out], tf.float32, initializer_gamma)
#     mean_ema = tf.get_variable("mean_ema", [n_out], tf.float32, initializer_zero, trainable=False)
#     var_ema = tf.get_variable("var_ema", [n_out], tf.float32, initializer_zero, trainable=False)
#     return beta, gamma, mean_ema, var_ema


import tensorflow as tf
import numpy as np
from tensorflow.python.training import moving_averages
from network.Util import create_batch_norm_vars
    summary = tf.summary.image(name, im)
    self.summaries.append(summary)

  def create_and_apply_batch_norm(self, inp, n_features, decay, tower_setup, scope_name="bn", freeze_batchnorm_override=None):
    beta, gamma, moving_mean, moving_var = create_batch_norm_vars(n_features, tower_setup, scope_name)
    self.n_params += 2 * n_features
    if tower_setup.is_main_train_tower:
      assert tower_setup.is_training
    if tower_setup.is_training and\
      (not tower_setup.freeze_batchnorm or (freeze_batchnorm_override is not None and freeze_batchnorm_override)):
        xn, batch_mean, batch_var = tf.nn.fused_batch_norm(inp, gamma, beta, epsilon=Layer.BATCH_NORM_EPSILON,
                                                           is_training=True)
        if tower_setup.is_main_train_tower:
          update_op1 = moving_averages.assign_moving_average(
            moving_mean, batch_mean, decay, zero_debias=False, name='mean_ema_op')
          update_op2 = moving_averages.assign_moving_average(
            moving_var, batch_var, decay, zero_debias=False, name='var_ema_op')
          self.update_ops.append(update_op1)
          self.update_ops.append(update_op2)
        return xn
    else:
      # According to the tensorpack code, this updates gamma and beta but not the moving averages
      # Fused version should have better performance.
      xn, _, _ = tf.nn.fused_batch_norm(inp, gamma, beta, moving_mean, moving_var, Layer.BATCH_NORM_EPSILON, is_training=False)
      return xn

  def create_weight_variable(self, name, shape, l2, tower_setup, trainable=True, initializer=None):
    with tf.device(tower_setup.variable_device):
      if initializer is None:
        # He initialization

# Path: network/Util.py
# def create_batch_norm_vars(n_out, tower_setup, scope_name="bn"):
#   with tf.device(tower_setup.variable_device), tf.variable_scope(scope_name):
#     initializer_zero = tf.constant_initializer(0.0, dtype=tf.float32)
#     beta = tf.get_variable("beta", [n_out], tf.float32, initializer_zero)
#     initializer_gamma = tf.constant_initializer(1.0, dtype=tf.float32)
#     gamma = tf.get_variable("gamma", [n_out], tf.float32, initializer_gamma)
#     mean_ema = tf.get_variable("mean_ema", [n_out], tf.float32, initializer_zero, trainable=False)
#     var_ema = tf.get_variable("var_ema", [n_out], tf.float32, initializer_zero, trainable=False)
#     return beta, gamma, mean_ema, var_ema


# Path: network/Layer.py
import tensorflow as tf
import numpy as np
from tensorflow.python.training import moving_averages
from network.Util import create_batch_norm_vars

    summary = tf.summary.image(name, im)
    self.summaries.append(summary)

  def create_and_apply_batch_norm(self, inp, n_features, decay, tower_setup, scope_name="bn", freeze_batchnorm_override=None):
    beta, gamma, moving_mean, moving_var = create_batch_norm_vars(n_features, tower_setup, scope_name)
    self.n_params += 2 * n_features
    if tower_setup.is_main_train_tower:
      assert tower_setup.is_training
    if tower_setup.is_training and\
      (not tower_setup.freeze_batchnorm or (freeze_batchnorm_override is not None and freeze_batchnorm_override)):
        xn, batch_mean, batch_var = tf.nn.fused_batch_norm(inp, gamma, beta, epsilon=Layer.BATCH_NORM_EPSILON,
                                                           is_training=True)
        if tower_setup.is_main_train_tower:
          update_op1 = moving_averages.assign_moving_average(
            moving_mean, batch_mean, decay, zero_debias=False, name='mean_ema_op')
          update_op2 = moving_averages.assign_moving_average(
            moving_var, batch_var, decay, zero_debias=False, name='var_ema_op')
          self.update_ops.append(update_op1)
          self.update_ops.append(update_op2)
        return xn
    else:
      # According to the tensorpack code, this updates gamma and beta but not the moving averages
      # Fused version should have better performance.
      xn, _, _ = tf.nn.fused_batch_norm(inp, gamma, beta, moving_mean, moving_var, Layer.BATCH_NORM_EPSILON, is_training=False)
      return xn

  def create_weight_variable(self, name, shape, l2, tower_setup, trainable=True, initializer=None):
    with tf.device(tower_setup.variable_device):
      if initializer is None:
        # He initialization

        initializer = tf.contrib.layers.variance_scaling_initializer(factor=2.0, mode='FAN_IN', uniform=False)