<jupyter_start><jupyter_text>Ops<jupyter_code>def prelu(x,name='prelu'):
    in_shape = x.get_shape().as_list()
    with tf.variable_scope(name):
        alpha = tf.get_variable('alpha',in_shape[-1],
                               initializer=tf.constant_initializer(0.),
                               dtype=tf.float32)
        pos = tf.nn.relu(x)
        neg = alpha * (x-tf.abs(x))*.5
        
        return pos + neg
def lrelu(x, alpha=0.3, name='lrelu'):
    return tf.maximum(x, alpha * x, name=name)
def bn(x,ri=[0,1,2],eps=.0001,scaling=True):
    xshp=x.get_shape().as_list()
    with tf.variable_scope('bn'):
        mu = tf.reduce_mean(x,ri,keep_dims=True)
        sigma = tf.reduce_mean(tf.square(x-mu),ri,keep_dims=True)
        x_hat = (x-mu)/(tf.sqrt(sigma+eps))
        
        if scaling:
            gamma = tf.get_variable('gamma',[xshp[-1]],initializer=tf.truncated_normal_initializer(0,.1))
            beta = tf.get_variable('beta',[xshp[-1]],initializer=tf.constant_initializer(1.))
            y = gamma*x_hat+beta
        else:
            y = x_hat
    return y
b = Batch(mapping=bin_mapping,bin_max=bin_size_x[1],batch_size=32)
save_path = 'modelckpt/disc_raw.ckpt'
tb_path = '../tb_logs/'<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>g = tf.Graph()
with g.as_default():
    x = tf.placeholder(tf.float32,shape=b.shape)
    y = tf.placeholder(tf.float32,shape=b.shape)
with g.as_default():
    def Dis(x,reuse=False):
        with tf.variable_scope('Discriminator') as scope:
            if reuse:
                scope.reuse_variables()
            #Down Colnvolutions
            #create variables
            with tf.variable_scope('in'):
                # Shape (height,width,inchannels,out_channels)
                #down convolutions operations
                with tf.variable_scope('l1'):
                    dis_conv1=tf.get_variable('conv1',shape=[1,31,1,1],initializer=tf.truncated_normal_initializer(0,.02))
                    dis_b1 = tf.get_variable('b1',shape=1,initializer=tf.constant_initializer(0.1))
                    dis_l1=lrelu(bn(tf.nn.conv2d(x,dis_conv1,strides=[1,1,2,1],padding='SAME')+dis_b1))
                with tf.variable_scope('l2'):
                    dis_conv2=tf.get_variable('conv2',shape=[1,31,1,16],initializer=tf.truncated_normal_initializer(0,.02))
                    dis_b2 = tf.get_variable('b2',shape=16,initializer=tf.constant_initializer(0.1))
                    dis_l2=lrelu(bn(tf.nn.conv2d(dis_l1,dis_conv2,strides=[1,1,2,1],padding='SAME')+dis_b2))
                with tf.variable_scope('l3'):
                    dis_conv3=tf.get_variable('conv3',shape=[1,31,16,32],initializer=tf.truncated_normal_initializer(0,.02))
                    dis_b3 = tf.get_variable('b3',shape=32,initializer=tf.constant_initializer(0.1))
                    dis_l3=lrelu(bn(tf.nn.conv2d(dis_l2,dis_conv3,strides=[1,1,2,1],padding='SAME')+dis_b3))
                with tf.variable_scope('l4'):
                    dis_conv4=tf.get_variable('conv4',shape=[1,31,32,32],initializer=tf.truncated_normal_initializer(0,.02))
                    dis_b4 = tf.get_variable('b4',shape=32,initializer=tf.constant_initializer(0.1))
                    dis_l4=lrelu(bn(tf.nn.conv2d(dis_l3,dis_conv4,strides=[1,1,2,1],padding='SAME')+dis_b4))
                with tf.variable_scope('l5'):
                    dis_conv5=tf.get_variable('conv5',shape=[1,31,32,64],initializer=tf.truncated_normal_initializer(0,.02))
                    dis_b5 = tf.get_variable('b5',shape=64,initializer=tf.constant_initializer(0.1))
                    dis_l5=lrelu(bn(tf.nn.conv2d(dis_l4,dis_conv5,strides=[1,1,2,1],padding='SAME')+dis_b5))
                with tf.variable_scope('l6'):
                    dis_conv6=tf.get_variable('conv6',shape=[1,31,64,64],initializer=tf.truncated_normal_initializer(0,.02))
                    dis_b6 = tf.get_variable('b6',shape=64,initializer=tf.constant_initializer(0.1))
                    dis_l6=lrelu(bn(tf.nn.conv2d(dis_l5,dis_conv6,strides=[1,1,2,1],padding='SAME')+dis_b6))
                with tf.variable_scope('l7'):
                    dis_conv7=tf.get_variable('conv7',shape=[1,31,64,128],initializer=tf.truncated_normal_initializer(0,.02))
                    dis_b7 = tf.get_variable('b7',shape=128,initializer=tf.constant_initializer(0.1))
                    dis_l7=lrelu(bn(tf.nn.conv2d(dis_l6,dis_conv7,strides=[1,1,2,1],padding='SAME')+dis_b7))

            dis_last_shape = dis_l7.get_shape()
            dis_es=int(dis_last_shape[2])*int(dis_last_shape[3]) #size of encoding
            dis_last = tf.reshape(bn(dis_l7),[b.batch_size,dis_es,1,1])
            
            with tf.variable_scope('final_conv'):
                dis_conv_na = tf.get_variable('conv',shape=[1,1,1,1],initializer=tf.truncated_normal_initializer(0,.02))
                dis_b_na = tf.get_variable('b',shape=[1],initializer=tf.constant_initializer(0.0))
                dis_na=tf.nn.conv2d(dis_last,dis_conv_na,strides=[1,1,1,1],padding='SAME')+dis_b_na
                dis_na = tf.reshape(dis_na,[b.batch_size,dis_es])
                

            #fully connected
            with tf.variable_scope('fc'):
                W_fc = tf.get_variable('W',shape=[dis_es,1],initializer=tf.truncated_normal_initializer(0.1))
                b_fc = tf.get_variable('b',shape=[1],initializer=tf.constant_initializer(0.1))

                #D_out = tf.sigmoid(bn(tf.matmul(dis_last,W_fc)+b_fc,ri=[0,1],scaling=True))
                D_out = tf.matmul(dis_na,W_fc)+b_fc
        return D_out<jupyter_output><empty_output><jupyter_text>Losses and optimization<jupyter_code>with g.as_default():
    x_logit=Dis(x)
    y_logit=Dis(y,reuse=True)
    
    x_pred = tf.sigmoid(x_logit)
    y_pred = tf.sigmoid(y_logit)
    
    #early losses
    loss1 = tf.reduce_mean(tf.square(x_logit)) #regularized loss for x
    loss2 = tf.reduce_mean(tf.square(y_logit-1)) #regularized loss for y
    loss_early = loss1+loss2
    
    #later
    loss3 = tf.reduce_mean(tf.log(x_pred))
    loss4 = tf.reduce_mean(tf.log(1-y_pred))
    loss_later = loss3+loss4

    #earlier optimizer
    opt = tf.train.AdamOptimizer(learning_rate=0.0002).minimize(loss_early)
    
    #late optimizer
#     dvars = [e for e in g.get_collection('trainable_variables') if 'Discriminator' in e.name]
#     # gradient clipping
#     optimizer = tf.train.AdadeltaOptimizer(learning_rate=.0002)
#     grad_d,var_d = zip(*optimizer.compute_gradients(loss_early,var_list=dvars))
#     grad_d_clipped ,_= tf.clip_by_global_norm(grad_d,.5)
#     opt_late=optimizer.apply_gradients(zip(grad_d_clipped,var_d))<jupyter_output><empty_output><jupyter_text>Summaries<jupyter_code>with g.as_default():
    #tf.summary.scalar('max_gradient',tf.maximum(tf.abs(grad_d)))
    tf.summary.scalar('Early_Discriminator_Loss',loss_early)
    tf.summary.scalar('Later_Discriminator_Loss',loss_later)
    tf.summary.histogram("X_Predictions",x_pred)
    tf.summary.histogram("Y_Predictions",y_pred)
    merged = tf.summary.merge_all()<jupyter_output><empty_output><jupyter_text>Initialize Variables<jupyter_code>with g.as_default():
    init = tf.global_variables_initializer()<jupyter_output><empty_output><jupyter_text>Saver<jupyter_code>with g.as_default():
    saver = tf.train.Saver()<jupyter_output><empty_output><jupyter_text>Session and writer<jupyter_code>sess=tf.Session(graph=g)
writer = tf.summary.FileWriter(tb_path,g)
sess.run(init)
losses = []
x_data,y_data = b.getBatch()
fd = {x:x_data,y:y_data}
sess.run(y_pred,feed_dict=fd)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>num_steps = 300
save_step = 500
for i in tqdm(xrange(num_steps)):
    x_data,y_data = b.getBatch()
    fd = {x:x_data,y:y_data}
    _,l = sess.run([opt,loss_early],feed_dict=fd)
#     _ = sess.run([opt_late],feed_dict=fd)
    losses.append(l)
    if i % 10 == 0:
        summary = sess.run(merged,feed_dict = fd)
        writer.add_summary(summary,i)
    if i % save_step == 0:
        save_path = saver.save(sess, save_path)
plt.plot(losses)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14783253056687662, 'fraction_numerical': 0.034333703840928737, 'mean_word_length': 3.3678532901833873, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '33220', 'license': 'MIT', 'n_tokens_mistral': 3180, 'n_tokens_neox': 3035, 'n_words': 273, 'path': 'SEGAN/Discriminator Raw.ipynb', 'repo_name': 'tmulc18/S2SCycleGAN', 'size': '8097'}