<commit_before>import tensorflow as tf
from lib.util.ops import *
from lib.util.globals import *

# This is openai's implementation of minibatch regularization
def get_features(config,net):
  net = tf.reshape(net, [config['batch_size']*2, -1])
  minis= get_minibatch_features(config, net, config['batch_size']*2,config['dtype'])
  print("MINIS", minis)
  return minis

def get_minibatch_features(config, h,batch_size,dtype):
  single_batch_size = batch_size//2
  n_kernels = int(config['d_kernels'])
  dim_per_kernel = int(config['d_kernel_dims'])
  print("Discriminator minibatch is projecting from", h, "to", n_kernels*dim_per_kernel)
  x = linear(h, n_kernels * dim_per_kernel, scope="d_h")
  x = batch_norm(config['batch_size'], name='d_bnend2')(x)
  activation = tf.reshape(x, (batch_size, n_kernels, dim_per_kernel))

  big = np.zeros((batch_size, batch_size))
  big += np.eye(batch_size)
  big = tf.expand_dims(big, 1)
  big = tf.cast(big,dtype=dtype)

  abs_dif = tf.reduce_sum(tf.abs(tf.expand_dims(activation,3) - tf.expand_dims(tf.transpose(activation, [1, 2, 0]), 0)), 2)
  mask = 1. - big
  masked = tf.exp(-abs_dif) * mask
  def half(tens, second):
    m, n, _ = tens.get_shape()
    m = int(m)
    n = int(n)
    return tf.slice(tens, [0, 0, second * single_batch_size], [m, n, single_batch_size])
  # TODO: speedup by allocating the denominator directly instead of constructing it by sum
  #       (current version makes it easier to play with the mask and not need to rederive
  #        the denominator)
  f1 = tf.reduce_sum(half(masked, 0), 2) / tf.reduce_sum(half(mask, 0))
  f2 = tf.reduce_sum(half(masked, 1), 2) / tf.reduce_sum(half(mask, 1))<commit_msg>[fix] remove erronous batch norm<commit_after>import tensorflow as tf
from lib.util.ops import *
from lib.util.globals import *

# This is openai's implementation of minibatch regularization
def get_features(config,net):
  net = tf.reshape(net, [config['batch_size']*2, -1])
  minis= get_minibatch_features(config, net, config['batch_size']*2,config['dtype'])
  print("MINIS", minis)
  return minis

def get_minibatch_features(config, h,batch_size,dtype):
  single_batch_size = batch_size//2
  n_kernels = int(config['d_kernels'])
  dim_per_kernel = int(config['d_kernel_dims'])
  print("Discriminator minibatch is projecting from", h, "to", n_kernels*dim_per_kernel)
  x = linear(h, n_kernels * dim_per_kernel, scope="d_h")
  activation = tf.reshape(x, (batch_size, n_kernels, dim_per_kernel))

  big = np.zeros((batch_size, batch_size))
  big += np.eye(batch_size)
  big = tf.expand_dims(big, 1)
  big = tf.cast(big,dtype=dtype)

  abs_dif = tf.reduce_sum(tf.abs(tf.expand_dims(activation,3) - tf.expand_dims(tf.transpose(activation, [1, 2, 0]), 0)), 2)
  mask = 1. - big
  masked = tf.exp(-abs_dif) * mask
  def half(tens, second):
    m, n, _ = tens.get_shape()
    m = int(m)
    n = int(n)
    return tf.slice(tens, [0, 0, second * single_batch_size], [m, n, single_batch_size])
  # TODO: speedup by allocating the denominator directly instead of constructing it by sum
  #       (current version makes it easier to play with the mask and not need to rederive
  #        the denominator)
  f1 = tf.reduce_sum(half(masked, 0), 2) / tf.reduce_sum(half(mask, 0))
  f2 = tf.reduce_sum(half(masked, 1), 2) / tf.reduce_sum(half(mask, 1))
{'fraction_non_alphanumeric': 0.13910681955340978, 'fraction_numerical': 0.013578756789378395, 'mean_word_length': 4.725388601036269, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6601272', 'n_tokens_mistral': 1272, 'n_tokens_neox': 1240, 'n_words': 292}