<jupyter_start><jupyter_text>In this notebook, we explore the benefits of adding depths separable convolutions to the legacy AnalogNet.  We train networks with growing number of depth-separable layers, compare their accuracy, and output their weights.This is done in view of eventually comparing their accuracy once implemented on the SCAMP5 camera, and demonstrate the noxious effects of noise.  * Legacy (one layer), reg then bin: 96.7% final testing accuracy (after FC retraining). * Legacy (one layer), bin then reg: 97.1% final testing accuracy (after FC retraining).  * One layer + one depth separable conv, bin then reg: 96.9% final testing accuracy with ReLU, 97.6% with lReLU0.2, 97.2% with lReLU0.25 (after FC retraining) (97.3% before rounding)  * One layer + two depth separable conv, bin then reg: 97.6% with lReLU.2 (97.9% before rounding + FC retrain), 97.62% with lReLU.25 (after FC retraining)BEWARE: checkpoint file names are interchanged between .2 and .25 ReLU versions... For this version of the network (one layer + 2 depth sep conv layers), all files with the name .2 are in fact referring to the version using .25 ReLU, and vice-versa  * One layer + three depth separable conv, bin then reg: 97.69% with lReLU.25 (after FC retraining)<jupyter_code>!nvidia-smi
EPOCHS = 50
BATCH_SIZE = 100
LR = 0.001

ROUNDING_STEP_CONV = 0.25
ROUNDING_STEP_BIAS = 1.
REG_CONSTANT = 4.

MAX_BIN_RATE = 10

BATCH_SIZE_FC = 50
LR_FC = 0.0001
EPOCHS_FC = 100<jupyter_output><empty_output><jupyter_text>0 Import data and utils<jupyter_code>%matplotlib inline
import tensorflow as tf
import tensorflow.contrib.slim as slim
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import pickle
mnist = tf.keras.datasets.mnist

(x_train, y_train),(x_test, y_test) = mnist.load_data()

# Simulate an input binarization
x_train = np.minimum(x_train, 100) // 100 * 120
x_test = np.minimum(x_test, 100) // 100 * 120
# Sigmoids, whose steepness and bias can be adjusted
# A very steep sigmoid (high bin_rate) simulates a binarization
MAX_BIN_RATE = 50
def binarize_tensor_differentiable(input, thresh, bin_rate):
  out1 = tf.nn.sigmoid(bin_rate*(input[...,:1] - thresh[0]))
  out2 = tf.nn.sigmoid(bin_rate*(input[...,1:2] - thresh[1]))
  out3 = tf.nn.sigmoid(bin_rate*(input[...,2:3] - thresh[2]))
  return tf.concat([out1, out2, out3], axis=-1)
def test_accuracy_bin_rate(bin_rate_feed):
  accs = np.zeros(x_test.shape[0] // BATCH_SIZE)
  for i in range(x_test.shape[0] // BATCH_SIZE):
    start = i * BATCH_SIZE
    stop = start + BATCH_SIZE
    
    xs = np.expand_dims(x_test[start:stop],-1)
    ys = y_test[start:stop]
    
    current_acc = sess.run(acc_op,
                       feed_dict={in_image_ph: xs,
                                  gt_label_ph: ys,
                                  bin_rate_ph: bin_rate_feed})
    accs[i] = current_acc
  
  print('Testing Acc.: {}'.format(
        accs.mean()))
def test_accuracy_bin_rate_fc(bin_rate_feed, verbose=False):
  accs = np.zeros(x_test.shape[0] // BATCH_SIZE_FC)
  for i in range(x_test.shape[0] // BATCH_SIZE_FC):
    start = i * BATCH_SIZE_FC
    stop = start + BATCH_SIZE_FC
    
    xs = np.expand_dims(x_test[start:stop],-1)
    ys = y_test[start:stop]
    
    current_acc = sess.run(acc_op,
                       feed_dict={in_image_ph: xs,
                                  gt_label_ph: ys,
                                  bin_rate_ph: bin_rate_feed})
    accs[i] = current_acc
  
  if verbose:
    print('Testing Acc.: {}'.format(
          accs.mean()))
  return accs.mean()
def custom_pooling(conv_bin):
  sum1 = tf.reduce_sum(conv_bin[:,5:14,0:9,:], axis=[1,2])
  sum2 = tf.reduce_sum(conv_bin[:,14:23,0:9,:], axis=[1,2])
  sum3 = tf.reduce_sum(conv_bin[:,0:9,5:14,:], axis=[1,2])
  sum4 = tf.reduce_sum(conv_bin[:,5:14,5:14,:], axis=[1,2])
  sum5 = tf.reduce_sum(conv_bin[:,14:23,5:14,:], axis=[1,2])
  sum6 = tf.reduce_sum(conv_bin[:,19:28,5:14,:], axis=[1,2])
  sum7 = tf.reduce_sum(conv_bin[:,0:9,14:23,:], axis=[1,2])
  sum8 = tf.reduce_sum(conv_bin[:,5:14,14:23,:], axis=[1,2])
  sum9 = tf.reduce_sum(conv_bin[:,14:23,14:23,:], axis=[1,2])
  sum10 = tf.reduce_sum(conv_bin[:,19:28,14:23,:], axis=[1,2])
  sum11 = tf.reduce_sum(conv_bin[:,5:14,19:28,:], axis=[1,2])
  sum12 = tf.reduce_sum(conv_bin[:,14:23,19:28,:], axis=[1,2])
  
  pool = tf.concat([sum1, sum2, sum3, sum4, sum5, sum6, 
                       sum7, sum8, sum9, sum10, sum11, sum12], axis=1)
  
  return pool<jupyter_output><empty_output><jupyter_text>1 Legacy AnalogNet architecture  1.1 Network definition<jupyter_code># Define the network
def net1(input, thresh_ph, bin_rate_ph):
  # Convolution
  conv = slim.conv2d(input, 3, [3, 3], rate=1, activation_fn=None,
                     padding='SAME', scope='conv1')
  
  # Output thresholding
  conv_bin = binarize_tensor_differentiable(conv, thresh_ph, bin_rate_ph)
  
  # Custom pooling (central division + sum overlapping)
  pool = custom_pooling(conv_bin)
  
  # Flatten + dense
  flat = tf.layers.flatten(pool)
  dense = tf.layers.dense(flat, 50, name='dense1', activation=tf.nn.relu)
  out = tf.layers.dense(dense, 10, name='dense2')
  
  return out
# Define the graph
tf.reset_default_graph()

in_image_ph = tf.placeholder(tf.float32, [BATCH_SIZE,28,28,1])
gt_label_ph = tf.placeholder(tf.uint8)
thresh = tf.Variable(tf.random.normal([3]), name='out_thresholds')
bin_rate_ph = tf.placeholder(tf.float32, ())

out_label_op = net1(in_image_ph, thresh, bin_rate_ph)

pred_op = tf.dtypes.cast(
            tf.keras.backend.argmax(out_label_op),
            tf.uint8)

loss_op = tf.reduce_mean(
          tf.keras.backend.sparse_categorical_crossentropy(gt_label_ph,
                                                           out_label_op,
                                                           from_logits=True))

acc_op = tf.contrib.metrics.accuracy(gt_label_ph, pred_op)
# Create session
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Define regulariser
def customRegularizerConv(x):
  return tf.math.cos(2/ROUNDING_STEP_CONV*np.pi*(x-ROUNDING_STEP_CONV/2))+1.

def customRegularizerBias(x):
  return tf.math.cos(2/ROUNDING_STEP_BIAS*np.pi*(x-ROUNDING_STEP_BIAS/2))+1.

with tf.variable_scope('conv1', reuse=True) as scope_conv:
  w1 = tf.get_variable('weights')

NUM_PARAM_REG = 3*3*3
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w1))

reg_factor_ph = tf.placeholder(tf.float32)
loss_with_reg_op = loss_op + reg_factor_ph * reg_losses

lr_ph = tf.placeholder(tf.float32)
opt = tf.train.AdamOptimizer(learning_rate=lr_ph, name='Adam_reg')
opt_op = opt.minimize(loss_with_reg_op)
sess.run(tf.variables_initializer(opt.variables()))<jupyter_output><empty_output><jupyter_text>1.2 Training<jupyter_code>"""
# Reg then bin
for epoch in range(EPOCHS*12):
  if epoch < EPOCHS * 4:
    lr_feed = LR
    reg_factor_feed = 0.
    bin_rate_feed = 1.
  elif epoch < EPOCHS * 8:
    lr_feed = LR / 2.
    reg_factor_feed = REG_CONSTANT
    bin_rate_feed = 1.
  else:
    lr_feed = LR / 4.
    reg_factor_feed = REG_CONSTANT
    bin_rate_feed = MAX_BIN_RATE + 1
    
# Bin then reg
for epoch in range(EPOCHS*12):
  if epoch < EPOCHS * 4:
    lr_feed = LR
    reg_factor_feed = 0.
    bin_rate_feed = 1.
  elif epoch < EPOCHS * 8:
    lr_feed = LR / 2.
    reg_factor_feed = 0.
    bin_rate_feed = MAX_BIN_RATE + 1
  else:
    lr_feed = LR / 4.
    reg_factor_feed = REG_CONSTANT
    bin_rate_feed = MAX_BIN_RATE + 1
"""
# Bin then reg
for epoch in range(EPOCHS*12):
  if epoch < EPOCHS * 4:
    lr_feed = LR
    reg_factor_feed = 0.
    bin_rate_feed = 1.
  elif epoch < EPOCHS * 8:
    lr_feed = LR / 2.
    reg_factor_feed = 0.
    bin_rate_feed = MAX_BIN_RATE + 1
  else:
    lr_feed = LR / 4.
    reg_factor_feed = REG_CONSTANT
    bin_rate_feed = MAX_BIN_RATE + 1
    
  random_perm = np.random.permutation(x_train.shape[0])
  losses = np.zeros(x_train.shape[0] // BATCH_SIZE)
  for i in range(x_train.shape[0] // BATCH_SIZE):
    start = i * BATCH_SIZE
    stop = start + BATCH_SIZE
    selected = random_perm[start:stop]
    
    xs = np.expand_dims(x_train[selected],-1)
    ys = y_train[selected]
    
    _, current_loss = sess.run([opt_op, loss_op],
                       feed_dict={in_image_ph: xs,
                                  gt_label_ph: ys,
                                  lr_ph: lr_feed,
                                  reg_factor_ph: reg_factor_feed,
                                  bin_rate_ph: bin_rate_feed})

    losses[i] = current_loss
  
  print('Epoch {} completed, average training loss is {}'.format(
          epoch+1, losses.mean()))
  test_accuracy_bin_rate(bin_rate_feed)
test_accuracy_bin_rate(bin_rate_feed)
# Save models, for not having to fully retrain it each time.

# Testing Acc.: 0.9635000038146972
saver = tf.train.Saver()
saver.save(sess, 'one_layer_bin_reg/model.ckpt')
!zip -r one_layer_bin_reg.zip one_layer_bin_reg<jupyter_output>adding: one_layer_bin_reg/ (stored 0%)
  adding: one_layer_bin_reg/.ipynb_checkpoints/ (stored 0%)
  adding: one_layer_bin_reg/checkpoint (deflated 42%)
  adding: one_layer_bin_reg/model.ckpt.index (deflated 40%)
  adding: one_layer_bin_reg/model.ckpt.data-00000-of-00001 (deflated 11%)
  adding: one_layer_bin_reg/model.ckpt.meta (deflated 88%)<jupyter_text>1.3 Re-training FC after rounding the conv kernels<jupyter_code>!unzip one_layer_bin_reg.zip<jupyter_output>Archive:  one_layer_bin_reg.zip
   creating: one_layer_bin_reg/
   creating: one_layer_bin_reg/.ipynb_checkpoints/
  inflating: one_layer_bin_reg/checkpoint  
  inflating: one_layer_bin_reg/model.ckpt.index  
  inflating: one_layer_bin_reg/model.ckpt.data-00000-of-00001  
  inflating: one_layer_bin_reg/model.ckpt.meta<jupyter_text>1.3.1 Reload Network<jupyter_code># Define the graph
tf.reset_default_graph()

in_image_ph = tf.placeholder(tf.float32, [BATCH_SIZE_FC,28,28,1])
gt_label_ph = tf.placeholder(tf.uint8)
thresh = tf.Variable(tf.random.normal([3]), name='out_thresholds')
bin_rate_ph = tf.placeholder(tf.float32, ())

out_label_op = net1(in_image_ph, thresh, bin_rate_ph)

pred_op = tf.dtypes.cast(
            tf.keras.backend.argmax(out_label_op),
            tf.uint8)

loss_op = tf.reduce_mean(
          tf.keras.backend.sparse_categorical_crossentropy(gt_label_ph,
                                                           out_label_op,
                                                           from_logits=True))

acc_op = tf.contrib.metrics.accuracy(gt_label_ph, pred_op)

saver = tf.train.Saver()
sess = tf.Session()
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state('one_layer_bin_reg')
saver.restore(sess, ckpt.model_checkpoint_path)
test_accuracy_bin_rate_fc(MAX_BIN_RATE)<jupyter_output><empty_output><jupyter_text>1.3.2 Round weights<jupyter_code>with tf.variable_scope('conv1', reuse=True) as scope_conv:
  w1 = tf.get_variable('weights')
# Rounding operation
rounding_weights1_op = tf.assign(w1, 
                          tf.round(w1/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)

_ = sess.run(rounding_weights1_op)

test_accuracy_bin_rate_fc(MAX_BIN_RATE)

# Show final distribution of weights
w1_values = sess.run(w1)

kernel_values = (list(w1_values.flatten()))
fig = plt.figure()
plt.scatter(kernel_values, [1]*len(kernel_values))<jupyter_output><empty_output><jupyter_text>1.3.3 Retrain FC only<jupyter_code>with tf.variable_scope('dense1', reuse=True) as scope_conv:
  fc1_k = tf.get_variable('kernel')
  fc1_b = tf.get_variable('bias')

with tf.variable_scope('dense2', reuse=True) as scope_conv:
  fc2_k = tf.get_variable('kernel')
  fc2_b = tf.get_variable('bias')

lr_ph = tf.placeholder(tf.float32)
opt_fc = tf.train.AdamOptimizer(learning_rate=lr_ph, name='Adam_reg')
opt_fc_op = opt_fc.minimize(loss_op, var_list=[fc1_k, fc1_b,
                                               fc2_k, fc2_b])

sess.run(tf.variables_initializer(opt_fc.variables()))
# Very long training:
bin_rate_feed = MAX_BIN_RATE + 1.
max_test_accuracy = 0.
for epoch in range(EPOCHS_FC*10):
  if epoch < EPOCHS_FC:
    lr_feed = LR_FC*8
  elif epoch < 3*EPOCHS_FC:
    lr_feed = LR_FC * 4
  elif epoch < 6*EPOCHS_FC:
    lr_feed = LR_FC * 2
  elif epoch < 8:
    lr_feed = LR_FC
  elif epoch < 9:
    lr_feed = LR_FC / 2.
  else:
    lr_feed = LR_FC / 5.
  random_perm = np.random.permutation(x_train.shape[0])
  losses = np.zeros(x_train.shape[0] // BATCH_SIZE_FC)
  for i in range(x_train.shape[0] // BATCH_SIZE_FC):
    start = i * BATCH_SIZE_FC
    stop = start + BATCH_SIZE_FC
    selected = random_perm[start:stop]
    
    xs = np.expand_dims(x_train[selected],-1)
    ys = y_train[selected]
    
    _, current_loss = sess.run([opt_fc_op, loss_op],
                       feed_dict={in_image_ph: xs,
                                  gt_label_ph: ys,
                                  bin_rate_ph: bin_rate_feed,
                                  lr_ph: lr_feed})

    losses[i] = current_loss
  
  current_test_accuracy = test_accuracy_bin_rate_fc(bin_rate_feed)
  # Save best model
  if current_test_accuracy > max_test_accuracy:
    saver.save(sess, 'one_layer_best_fc_retrain/model.ckpt')
    max_test_accuracy = current_test_accuracy
  
  print('Epoch {} completed, average training loss is {}'.format(
          epoch+1, losses.mean()))
  print('Testing Acc.: {}'.format(current_test_accuracy))
  
# Restore best model
ckpt = tf.train.get_checkpoint_state('one_layer_best_fc_retrain')
saver.restore(sess, ckpt.model_checkpoint_path)
_ = test_accuracy_bin_rate_fc(bin_rate_feed, verbose=True)
_ = test_accuracy_bin_rate_fc(bin_rate_feed, verbose=True)<jupyter_output>Testing Acc.: 0.9715000021457673<jupyter_text>1.4 Extract conv weights<jupyter_code>w1_values
sess.run(thresh)
with open('NP_WEIGHTS_one_layer_bin_reg.pck', 'wb') as f:
  pickle.dump(w1_values, f)<jupyter_output><empty_output><jupyter_text>2 Add 1 depth separable conv layer  2.1 Network definition<jupyter_code>def lrelu(x):
    return tf.maximum(x * 0.25, x)
# Define the network
def net2(input, thresh_ph, bin_rate_ph):
  # Convolution
  #conv = slim.conv2d(input, 3, [3, 3], rate=1, activation_fn=tf.nn.relu,
  #                   padding='SAME', scope='conv1')
  conv = slim.conv2d(input, 3, [3, 3], rate=1, activation_fn=lrelu,
                     padding='SAME', scope='conv1')
  
  # One depth separable conv
  conv_2_1 = slim.conv2d(conv[...,:1], 1, [3, 3], rate=1, activation_fn=None,
                     padding='SAME', scope='conv2_1')
  conv_2_2 = slim.conv2d(conv[...,1:2], 1, [3, 3], rate=1, activation_fn=None,
                   padding='SAME', scope='conv2_2')
  conv_2_3 = slim.conv2d(conv[...,2:3], 1, [3, 3], rate=1, activation_fn=None,
                 padding='SAME', scope='conv2_3')
  
  conv2 = tf.concat([conv_2_1, conv_2_2, conv_2_3], axis=-1)
  
  # Output thresholding
  conv_bin = binarize_tensor_differentiable(conv2, thresh_ph, bin_rate_ph)
  
  # Custom pooling (central division + sum overlapping)
  pool = custom_pooling(conv_bin)
  
  # Flatten + dense
  flat = tf.layers.flatten(pool)
  dense = tf.layers.dense(flat, 50, name='dense1', activation=tf.nn.relu)
  out = tf.layers.dense(dense, 10, name='dense2')
  
  return out
# Define the graph
tf.reset_default_graph()

in_image_ph = tf.placeholder(tf.float32, [BATCH_SIZE,28,28,1])
gt_label_ph = tf.placeholder(tf.uint8)
thresh = tf.Variable(tf.random.normal([3]), name='out_thresholds')
bin_rate_ph = tf.placeholder(tf.float32, ())

out_label_op = net2(in_image_ph, thresh, bin_rate_ph)

pred_op = tf.dtypes.cast(
            tf.keras.backend.argmax(out_label_op),
            tf.uint8)

loss_op = tf.reduce_mean(
          tf.keras.backend.sparse_categorical_crossentropy(gt_label_ph,
                                                           out_label_op,
                                                           from_logits=True))

acc_op = tf.contrib.metrics.accuracy(gt_label_ph, pred_op)
# Create session
sess = tf.Session()
sess.run(tf.global_variables_initializer())
#[n.name for n in tf.get_default_graph().as_graph_def().node]
# Define regulariser
def customRegularizerConv(x):
  return tf.math.cos(2/ROUNDING_STEP_CONV*np.pi*(x-ROUNDING_STEP_CONV/2))+1.

def customRegularizerBias(x):
  return tf.math.cos(2/ROUNDING_STEP_BIAS*np.pi*(x-ROUNDING_STEP_BIAS/2))+1.

with tf.variable_scope('conv1', reuse=True) as scope_conv:
  w1 = tf.get_variable('weights')
  b1 = tf.get_variable('biases')
with tf.variable_scope('conv2_1', reuse=True) as scope_conv:
  w2_1 = tf.get_variable('weights')
with tf.variable_scope('conv2_2', reuse=True) as scope_conv:
  w2_2 = tf.get_variable('weights')
with tf.variable_scope('conv2_3', reuse=True) as scope_conv:
  w2_3 = tf.get_variable('weights')

NUM_PARAM_REG = np.prod(([int(e) for e in w1.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in b1.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in w2_1.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in w2_2.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in w2_3.shape]))

reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w1))

reg_factor_ph = tf.placeholder(tf.float32)
loss_with_reg_op = loss_op + reg_factor_ph * reg_losses

lr_ph = tf.placeholder(tf.float32)
opt = tf.train.AdamOptimizer(learning_rate=lr_ph, name='Adam_reg')
opt_op = opt.minimize(loss_with_reg_op)
sess.run(tf.variables_initializer(opt.variables()))<jupyter_output><empty_output><jupyter_text>2.2 Training<jupyter_code>"""
# Reg then bin
for epoch in range(EPOCHS*12):
  if epoch < EPOCHS * 4:
    lr_feed = LR
    reg_factor_feed = 0.
    bin_rate_feed = 1.
  elif epoch < EPOCHS * 8:
    lr_feed = LR / 2.
    reg_factor_feed = REG_CONSTANT
    bin_rate_feed = 1.
  else:
    lr_feed = LR / 4.
    reg_factor_feed = REG_CONSTANT
    bin_rate_feed = MAX_BIN_RATE + 1
    
# Bin then reg
for epoch in range(EPOCHS*12):
  if epoch < EPOCHS * 4:
    lr_feed = LR
    reg_factor_feed = 0.
    bin_rate_feed = 1.
  elif epoch < EPOCHS * 8:
    lr_feed = LR / 2.
    reg_factor_feed = 0.
    bin_rate_feed = MAX_BIN_RATE + 1
  else:
    lr_feed = LR / 4.
    reg_factor_feed = REG_CONSTANT
    bin_rate_feed = MAX_BIN_RATE + 1
"""
# Bin then reg
for epoch in range(EPOCHS*12):
  if epoch < EPOCHS * 4:
    lr_feed = LR
    reg_factor_feed = 0.
    bin_rate_feed = 1.
  elif epoch < EPOCHS * 8:
    lr_feed = LR / 2.
    reg_factor_feed = 0.
    bin_rate_feed = MAX_BIN_RATE + 1
  else:
    lr_feed = LR / 4.
    reg_factor_feed = REG_CONSTANT
    bin_rate_feed = MAX_BIN_RATE + 1
    
  random_perm = np.random.permutation(x_train.shape[0])
  losses = np.zeros(x_train.shape[0] // BATCH_SIZE)
  for i in range(x_train.shape[0] // BATCH_SIZE):
    start = i * BATCH_SIZE
    stop = start + BATCH_SIZE
    selected = random_perm[start:stop]
    
    xs = np.expand_dims(x_train[selected],-1)
    ys = y_train[selected]
    
    _, current_loss = sess.run([opt_op, loss_op],
                       feed_dict={in_image_ph: xs,
                                  gt_label_ph: ys,
                                  lr_ph: lr_feed,
                                  reg_factor_ph: reg_factor_feed,
                                  bin_rate_ph: bin_rate_feed})

    losses[i] = current_loss
  
  print('Epoch {} completed, average training loss is {}'.format(
          epoch+1, losses.mean()))
  test_accuracy_bin_rate(bin_rate_feed)
  
test_accuracy_bin_rate(bin_rate_feed)
# Save models, for not having to fully retrain it each time.

# Testing Acc.: 0.9714000046253204 for bin reg
# 0.9679000043869018 for reg bin
saver = tf.train.Saver()
saver.save(sess, 'two_layers_bin_reg_lrelu025/model.ckpt')
!zip -r two_layers_bin_reg_lrelu025.zip two_layers_bin_reg_lrelu025<jupyter_output>adding: two_layers_bin_reg_lrelu025/ (stored 0%)
  adding: two_layers_bin_reg_lrelu025/model.ckpt.index (deflated 50%)
  adding: two_layers_bin_reg_lrelu025/model.ckpt.meta (deflated 89%)
  adding: two_layers_bin_reg_lrelu025/model.ckpt.data-00000-of-00001 (deflated 15%)
  adding: two_layers_bin_reg_lrelu025/checkpoint (deflated 42%)<jupyter_text>2.3 Re-training FC after rounding the conv kernels  2.3.1 Reload Network<jupyter_code>!unzip two_layers_bin_reg_lrelu.zip
# Define the graph
tf.reset_default_graph()

in_image_ph = tf.placeholder(tf.float32, [BATCH_SIZE_FC,28,28,1])
gt_label_ph = tf.placeholder(tf.uint8)
thresh = tf.Variable(tf.random.normal([3]), name='out_thresholds')
bin_rate_ph = tf.placeholder(tf.float32, ())

out_label_op = net2(in_image_ph, thresh, bin_rate_ph)

pred_op = tf.dtypes.cast(
            tf.keras.backend.argmax(out_label_op),
            tf.uint8)

loss_op = tf.reduce_mean(
          tf.keras.backend.sparse_categorical_crossentropy(gt_label_ph,
                                                           out_label_op,
                                                           from_logits=True))

acc_op = tf.contrib.metrics.accuracy(gt_label_ph, pred_op)

saver = tf.train.Saver()
sess = tf.Session()
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state('two_layers_bin_reg_lrelu')
saver.restore(sess, ckpt.model_checkpoint_path)
test_accuracy_bin_rate_fc(MAX_BIN_RATE)<jupyter_output><empty_output><jupyter_text>2.3.2 Round weights<jupyter_code>with tf.variable_scope('conv1', reuse=True) as scope_conv:
  w1 = tf.get_variable('weights')
  b1 = tf.get_variable('biases')
with tf.variable_scope('conv2_1', reuse=True) as scope_conv:
  w2_1 = tf.get_variable('weights')
with tf.variable_scope('conv2_2', reuse=True) as scope_conv:
  w2_2 = tf.get_variable('weights')
with tf.variable_scope('conv2_3', reuse=True) as scope_conv:
  w2_3 = tf.get_variable('weights')
# Rounding operation
rounding_weights1_op = tf.assign(w1, 
                          tf.round(w1/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)
rounding_bias1_op = tf.assign(b1, 
                          tf.round(b1/ROUNDING_STEP_BIAS)*ROUNDING_STEP_BIAS)
rounding_weights2_1_op = tf.assign(w2_1, 
                          tf.round(w2_1/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)
rounding_weights2_2_op = tf.assign(w2_2, 
                          tf.round(w2_2/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)
rounding_weights2_3_op = tf.assign(w2_3, 
                          tf.round(w2_3/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)

_ = sess.run(rounding_weights1_op)
_ = sess.run(rounding_bias1_op)
_ = sess.run(rounding_weights2_1_op)
_ = sess.run(rounding_weights2_2_op)
_ = sess.run(rounding_weights2_3_op)

test_accuracy_bin_rate_fc(MAX_BIN_RATE, verbose=True)

# Show final distribution of weights
w1_values = sess.run(w1)
b1_values = sess.run(b1)
w2_1_values = sess.run(w2_1)
w2_2_values = sess.run(w2_2)
w2_3_values = sess.run(w2_3)

kernel_values = (list(w1_values.flatten()) + 
                  list(b1_values.flatten()) +
                  list(w2_1_values.flatten()) +
                  list(w2_2_values.flatten()) +
                  list(w2_3_values.flatten()))
fig = plt.figure()
plt.scatter(kernel_values, [1]*len(kernel_values))<jupyter_output>Testing Acc.: 0.9437999960780143<jupyter_text>2.3.3 Retrain FC only<jupyter_code>with tf.variable_scope('dense1', reuse=True) as scope_conv:
  fc1_k = tf.get_variable('kernel')
  fc1_b = tf.get_variable('bias')

with tf.variable_scope('dense2', reuse=True) as scope_conv:
  fc2_k = tf.get_variable('kernel')
  fc2_b = tf.get_variable('bias')

lr_ph = tf.placeholder(tf.float32)
opt_fc = tf.train.AdamOptimizer(learning_rate=lr_ph, name='Adam_reg')
opt_fc_op = opt_fc.minimize(loss_op, var_list=[fc1_k, fc1_b,
                                               fc2_k, fc2_b])

sess.run(tf.variables_initializer(opt_fc.variables()))
# Very long training:
bin_rate_feed = MAX_BIN_RATE + 1
max_test_accuracy = 0.
for epoch in range(EPOCHS_FC*10):
  if epoch < EPOCHS_FC:
    lr_feed = LR_FC*8
  elif epoch < 3*EPOCHS_FC:
    lr_feed = LR_FC * 4
  elif epoch < 6*EPOCHS_FC:
    lr_feed = LR_FC * 2
  elif epoch < 8:
    lr_feed = LR_FC
  elif epoch < 9:
    lr_feed = LR_FC / 2.
  else:
    lr_feed = LR_FC / 5.
  random_perm = np.random.permutation(x_train.shape[0])
  losses = np.zeros(x_train.shape[0] // BATCH_SIZE_FC)
  for i in range(x_train.shape[0] // BATCH_SIZE_FC):
    start = i * BATCH_SIZE_FC
    stop = start + BATCH_SIZE_FC
    selected = random_perm[start:stop]
    
    xs = np.expand_dims(x_train[selected],-1)
    ys = y_train[selected]
    
    _, current_loss = sess.run([opt_fc_op, loss_op],
                       feed_dict={in_image_ph: xs,
                                  gt_label_ph: ys,
                                  bin_rate_ph: bin_rate_feed,
                                  lr_ph: lr_feed})

    losses[i] = current_loss
  
  current_test_accuracy = test_accuracy_bin_rate_fc(bin_rate_feed)
  # Save best model
  if current_test_accuracy > max_test_accuracy:
    saver.save(sess, 'one_layer_best_fc_retrain/model.ckpt')
    max_test_accuracy = current_test_accuracy
  
  print('Epoch {} completed, average training loss is {}'.format(
          epoch+1, losses.mean()))
  print('Testing Acc.: {}'.format(current_test_accuracy))
  
# Restore best model
ckpt = tf.train.get_checkpoint_state('one_layer_best_fc_retrain')
saver.restore(sess, ckpt.model_checkpoint_path)
_ = test_accuracy_bin_rate_fc(bin_rate_feed, verbose=True)

test_accuracy_bin_rate_fc(bin_rate_feed)<jupyter_output><empty_output><jupyter_text>2.4 Extract conv weights<jupyter_code>with open('NP_WEIGHTS_two_layers_bin_reg_lrelu.pck', 'wb') as f:
  pickle.dump((w1_values, b1_values, w2_1_values, w2_2_values, w2_3_values), f)<jupyter_output><empty_output><jupyter_text>3 Add 2 depth separable conv layers  3.1 Network definition<jupyter_code>def lrelu(x):
    return tf.maximum(x * 0.25, x)
# Define the network
def net3(input, thresh_ph, bin_rate_ph):
  # Convolution
  #conv = slim.conv2d(input, 3, [3, 3], rate=1, activation_fn=tf.nn.relu,
  #                   padding='SAME', scope='conv1')
  conv = slim.conv2d(input, 3, [3, 3], rate=1, activation_fn=lrelu,
                     padding='SAME', scope='conv1')
  
  # One depth separable conv
  conv_2_1 = slim.conv2d(conv[...,:1], 1, [3, 3], rate=1, activation_fn=lrelu,
                     padding='SAME', scope='conv2_1')
  conv_3_1 = slim.conv2d(conv_2_1, 1, [3, 3], rate=1, activation_fn=None,
                     padding='SAME', scope='conv3_1')
  
  conv_2_2 = slim.conv2d(conv[...,1:2], 1, [3, 3], rate=1, activation_fn=lrelu,
                     padding='SAME', scope='conv2_2')
  conv_3_2 = slim.conv2d(conv_2_2, 1, [3, 3], rate=1, activation_fn=None,
                     padding='SAME', scope='conv3_2')
  
  conv_2_3 = slim.conv2d(conv[...,2:3], 1, [3, 3], rate=1, activation_fn=lrelu,
                     padding='SAME', scope='conv2_3')
  conv_3_3 = slim.conv2d(conv_2_3, 1, [3, 3], rate=1, activation_fn=None,
                     padding='SAME', scope='conv3_3')
  
  
  conv3 = tf.concat([conv_3_1, conv_3_2, conv_3_3], axis=-1)
  
  # Output thresholding
  conv_bin = binarize_tensor_differentiable(conv3, thresh_ph, bin_rate_ph)
  
  # Custom pooling (central division + sum overlapping)
  pool = custom_pooling(conv_bin)
  
  # Flatten + dense
  flat = tf.layers.flatten(pool)
  dense = tf.layers.dense(flat, 50, name='dense1', activation=tf.nn.relu)
  out = tf.layers.dense(dense, 10, name='dense2')
  
  return out
# Define the graph
tf.reset_default_graph()

in_image_ph = tf.placeholder(tf.float32, [BATCH_SIZE,28,28,1])
gt_label_ph = tf.placeholder(tf.uint8)
thresh = tf.Variable(tf.random.normal([3]), name='out_thresholds')
bin_rate_ph = tf.placeholder(tf.float32, ())

out_label_op = net3(in_image_ph, thresh, bin_rate_ph)

pred_op = tf.dtypes.cast(
            tf.keras.backend.argmax(out_label_op),
            tf.uint8)

loss_op = tf.reduce_mean(
          tf.keras.backend.sparse_categorical_crossentropy(gt_label_ph,
                                                           out_label_op,
                                                           from_logits=True))

acc_op = tf.contrib.metrics.accuracy(gt_label_ph, pred_op)
# Create session
sess = tf.Session()
sess.run(tf.global_variables_initializer())
#[n.name for n in tf.get_default_graph().as_graph_def().node]
# Define regulariser
def customRegularizerConv(x):
  return tf.math.cos(2/ROUNDING_STEP_CONV*np.pi*(x-ROUNDING_STEP_CONV/2))+1.

def customRegularizerBias(x):
  return tf.math.cos(2/ROUNDING_STEP_BIAS*np.pi*(x-ROUNDING_STEP_BIAS/2))+1.

with tf.variable_scope('conv1', reuse=True) as scope_conv:
  w1 = tf.get_variable('weights')
  b1 = tf.get_variable('biases')
  
with tf.variable_scope('conv2_1', reuse=True) as scope_conv:
  w2_1 = tf.get_variable('weights')
  b2_1 = tf.get_variable('biases')
with tf.variable_scope('conv2_2', reuse=True) as scope_conv:
  w2_2 = tf.get_variable('weights')
  b2_2 = tf.get_variable('biases')
with tf.variable_scope('conv2_3', reuse=True) as scope_conv:
  w2_3 = tf.get_variable('weights')
  b2_3 = tf.get_variable('biases')
  
with tf.variable_scope('conv3_1', reuse=True) as scope_conv:
  w3_1 = tf.get_variable('weights')
with tf.variable_scope('conv3_2', reuse=True) as scope_conv:
  w3_2 = tf.get_variable('weights')
with tf.variable_scope('conv3_3', reuse=True) as scope_conv:
  w3_3 = tf.get_variable('weights')

NUM_PARAM_REG = np.prod(([int(e) for e in w1.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in b1.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in w2_1.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in b2_1.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in w2_2.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in b2_2.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in w2_3.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in b2_3.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in w3_1.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in w3_2.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in w3_3.shape]))

reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w1))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerBias(b1))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w2_1))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerBias(b2_1))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w2_2))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerBias(b2_2))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w2_3))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerBias(b2_3))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w3_1))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w3_2))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w3_3))

reg_factor_ph = tf.placeholder(tf.float32)
loss_with_reg_op = loss_op + reg_factor_ph * reg_losses

lr_ph = tf.placeholder(tf.float32)
opt = tf.train.AdamOptimizer(learning_rate=lr_ph, name='Adam_reg')
opt_op = opt.minimize(loss_with_reg_op)
sess.run(tf.variables_initializer(opt.variables()))<jupyter_output><empty_output><jupyter_text>3.2 Training<jupyter_code>"""
# Reg then bin
for epoch in range(EPOCHS*12):
  if epoch < EPOCHS * 4:
    lr_feed = LR
    reg_factor_feed = 0.
    bin_rate_feed = 1.
  elif epoch < EPOCHS * 8:
    lr_feed = LR / 2.
    reg_factor_feed = REG_CONSTANT
    bin_rate_feed = 1.
  else:
    lr_feed = LR / 4.
    reg_factor_feed = REG_CONSTANT
    bin_rate_feed = MAX_BIN_RATE + 1
    
# Bin then reg
for epoch in range(EPOCHS*12):
  if epoch < EPOCHS * 4:
    lr_feed = LR
    reg_factor_feed = 0.
    bin_rate_feed = 1.
  elif epoch < EPOCHS * 8:
    lr_feed = LR / 2.
    reg_factor_feed = 0.
    bin_rate_feed = MAX_BIN_RATE + 1
  else:
    lr_feed = LR / 4.
    reg_factor_feed = REG_CONSTANT
    bin_rate_feed = MAX_BIN_RATE + 1
"""
# Bin then reg
for epoch in range(EPOCHS*12):
  if epoch < EPOCHS * 4:
    lr_feed = LR
    reg_factor_feed = 0.
    bin_rate_feed = 1.
  elif epoch < EPOCHS * 8:
    lr_feed = LR / 2.
    reg_factor_feed = 0.
    bin_rate_feed = MAX_BIN_RATE + 1
  else:
    lr_feed = LR / 4.
    reg_factor_feed = REG_CONSTANT
    bin_rate_feed = MAX_BIN_RATE + 1
    
  random_perm = np.random.permutation(x_train.shape[0])
  losses = np.zeros(x_train.shape[0] // BATCH_SIZE)
  for i in range(x_train.shape[0] // BATCH_SIZE):
    start = i * BATCH_SIZE
    stop = start + BATCH_SIZE
    selected = random_perm[start:stop]
    
    xs = np.expand_dims(x_train[selected],-1)
    ys = y_train[selected]
    
    _, current_loss = sess.run([opt_op, loss_op],
                       feed_dict={in_image_ph: xs,
                                  gt_label_ph: ys,
                                  lr_ph: lr_feed,
                                  reg_factor_ph: reg_factor_feed,
                                  bin_rate_ph: bin_rate_feed})

    losses[i] = current_loss
  
  print('Epoch {} completed, average training loss is {}'.format(
          epoch+1, losses.mean()))
  test_accuracy_bin_rate(bin_rate_feed)
  
test_accuracy_bin_rate(bin_rate_feed)
# Save models, for not having to fully retrain it each time.

# Testing Acc.: 0.9714000046253204 for bin reg
# 0.9679000043869018 for reg bin
saver = tf.train.Saver()
saver.save(sess, 'three_layers_bin_reg_lrelu02/model.ckpt')
!zip -r three_layers_bin_reg_lrelu02.zip three_layers_bin_reg_lrelu02<jupyter_output>adding: three_layers_bin_reg_lrelu02/ (stored 0%)
  adding: three_layers_bin_reg_lrelu02/model.ckpt.meta (deflated 90%)
  adding: three_layers_bin_reg_lrelu02/model.ckpt.data-00000-of-00001 (deflated 31%)
  adding: three_layers_bin_reg_lrelu02/checkpoint (deflated 42%)
  adding: three_layers_bin_reg_lrelu02/model.ckpt.index (deflated 56%)<jupyter_text>3.3 Re-training FC after rounding the conv kernels  3.3.1 Reload Network<jupyter_code>!unzip three_layers_bin_reg_lrelu02.zip
# Define the graph
tf.reset_default_graph()

in_image_ph = tf.placeholder(tf.float32, [BATCH_SIZE_FC,28,28,1])
gt_label_ph = tf.placeholder(tf.uint8)
thresh = tf.Variable(tf.random.normal([3]), name='out_thresholds')
bin_rate_ph = tf.placeholder(tf.float32, ())

out_label_op = net3(in_image_ph, thresh, bin_rate_ph)

pred_op = tf.dtypes.cast(
            tf.keras.backend.argmax(out_label_op),
            tf.uint8)

loss_op = tf.reduce_mean(
          tf.keras.backend.sparse_categorical_crossentropy(gt_label_ph,
                                                           out_label_op,
                                                           from_logits=True))

acc_op = tf.contrib.metrics.accuracy(gt_label_ph, pred_op)

saver = tf.train.Saver()
sess = tf.Session()
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state('three_layers_bin_reg_lrelu02')
saver.restore(sess, ckpt.model_checkpoint_path)
test_accuracy_bin_rate_fc(MAX_BIN_RATE)<jupyter_output><empty_output><jupyter_text>3.3.2 Round weights<jupyter_code>with tf.variable_scope('conv1', reuse=True) as scope_conv:
  w1 = tf.get_variable('weights')
  b1 = tf.get_variable('biases')
  
with tf.variable_scope('conv2_1', reuse=True) as scope_conv:
  w2_1 = tf.get_variable('weights')
  b2_1 = tf.get_variable('biases')
with tf.variable_scope('conv2_2', reuse=True) as scope_conv:
  w2_2 = tf.get_variable('weights')
  b2_2 = tf.get_variable('biases')
with tf.variable_scope('conv2_3', reuse=True) as scope_conv:
  w2_3 = tf.get_variable('weights')
  b2_3 = tf.get_variable('biases')
  
with tf.variable_scope('conv3_1', reuse=True) as scope_conv:
  w3_1 = tf.get_variable('weights')
with tf.variable_scope('conv3_2', reuse=True) as scope_conv:
  w3_2 = tf.get_variable('weights')
with tf.variable_scope('conv3_3', reuse=True) as scope_conv:
  w3_3 = tf.get_variable('weights')
# Rounding operation
rounding_weights1_op = tf.assign(w1, 
                          tf.round(w1/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)
rounding_bias1_op = tf.assign(b1, 
                          tf.round(b1/ROUNDING_STEP_BIAS)*ROUNDING_STEP_BIAS)
rounding_weights2_1_op = tf.assign(w2_1, 
                          tf.round(w2_1/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)
rounding_bias2_1_op = tf.assign(b2_1, 
                          tf.round(b2_1/ROUNDING_STEP_BIAS)*ROUNDING_STEP_BIAS)
rounding_weights2_2_op = tf.assign(w2_2, 
                          tf.round(w2_2/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)
rounding_bias2_2_op = tf.assign(b2_2, 
                          tf.round(b2_2/ROUNDING_STEP_BIAS)*ROUNDING_STEP_BIAS)
rounding_weights2_3_op = tf.assign(w2_3, 
                          tf.round(w2_3/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)
rounding_bias2_3_op = tf.assign(b2_3, 
                          tf.round(b2_3/ROUNDING_STEP_BIAS)*ROUNDING_STEP_BIAS)
rounding_weights3_1_op = tf.assign(w3_1, 
                          tf.round(w3_1/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)
rounding_weights3_2_op = tf.assign(w3_2, 
                          tf.round(w3_2/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)
rounding_weights3_3_op = tf.assign(w3_3, 
                          tf.round(w3_3/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)

_ = sess.run(rounding_weights1_op)
_ = sess.run(rounding_bias1_op)
_ = sess.run(rounding_weights2_1_op)
_ = sess.run(rounding_bias2_1_op)
_ = sess.run(rounding_weights2_2_op)
_ = sess.run(rounding_bias2_2_op)
_ = sess.run(rounding_weights2_3_op)
_ = sess.run(rounding_bias2_3_op)

_ = sess.run(rounding_weights3_1_op)
_ = sess.run(rounding_weights3_2_op)
_ = sess.run(rounding_weights3_3_op)

test_accuracy_bin_rate_fc(MAX_BIN_RATE, verbose=True)

# Show final distribution of weights
w1_values = sess.run(w1)
b1_values = sess.run(b1)
w2_1_values = sess.run(w2_1)
b2_1_values = sess.run(b2_1)
w2_2_values = sess.run(w2_2)
b2_2_values = sess.run(b2_2)
w2_3_values = sess.run(w2_3)
b2_3_values = sess.run(b2_3)
w3_1_values = sess.run(w3_1)
w3_2_values = sess.run(w3_2)
w3_3_values = sess.run(w3_3)

kernel_values = (list(w1_values.flatten()) + 
                  list(b1_values.flatten()) +
                  list(w2_1_values.flatten()) +
                  list(b2_1_values.flatten()) +
                  list(w2_2_values.flatten()) +
                  list(b2_2_values.flatten()) +
                  list(w2_3_values.flatten()) +
                  list(b2_3_values.flatten()) +
                  list(w3_1_values.flatten()) +
                  list(w3_2_values.flatten()) +
                  list(w3_3_values.flatten()))
fig = plt.figure()
plt.scatter(kernel_values, [1]*len(kernel_values))<jupyter_output>Testing Acc.: 0.47669999942183494<jupyter_text>3.3.3 Retrain FC only<jupyter_code>with tf.variable_scope('dense1', reuse=True) as scope_conv:
  fc1_k = tf.get_variable('kernel')
  fc1_b = tf.get_variable('bias')

with tf.variable_scope('dense2', reuse=True) as scope_conv:
  fc2_k = tf.get_variable('kernel')
  fc2_b = tf.get_variable('bias')

lr_ph = tf.placeholder(tf.float32)
opt_fc = tf.train.AdamOptimizer(learning_rate=lr_ph, name='Adam_reg')
opt_fc_op = opt_fc.minimize(loss_op, var_list=[fc1_k, fc1_b,
                                               fc2_k, fc2_b])

sess.run(tf.variables_initializer(opt_fc.variables()))
# Very long training:
bin_rate_feed = MAX_BIN_RATE + 1
max_test_accuracy = 0.
for epoch in range(EPOCHS_FC*10):
  if epoch < EPOCHS_FC:
    lr_feed = LR_FC*8
  elif epoch < 3*EPOCHS_FC:
    lr_feed = LR_FC * 4
  elif epoch < 6*EPOCHS_FC:
    lr_feed = LR_FC * 2
  elif epoch < 8:
    lr_feed = LR_FC
  elif epoch < 9:
    lr_feed = LR_FC / 2.
  else:
    lr_feed = LR_FC / 5.
  random_perm = np.random.permutation(x_train.shape[0])
  losses = np.zeros(x_train.shape[0] // BATCH_SIZE_FC)
  for i in range(x_train.shape[0] // BATCH_SIZE_FC):
    start = i * BATCH_SIZE_FC
    stop = start + BATCH_SIZE_FC
    selected = random_perm[start:stop]
    
    xs = np.expand_dims(x_train[selected],-1)
    ys = y_train[selected]
    
    _, current_loss = sess.run([opt_fc_op, loss_op],
                       feed_dict={in_image_ph: xs,
                                  gt_label_ph: ys,
                                  bin_rate_ph: bin_rate_feed,
                                  lr_ph: lr_feed})

    losses[i] = current_loss
  
  current_test_accuracy = test_accuracy_bin_rate_fc(bin_rate_feed)
  # Save best model
  if current_test_accuracy > max_test_accuracy:
    saver.save(sess, 'one_layer_best_fc_retrain/model.ckpt')
    max_test_accuracy = current_test_accuracy
  
  print('Epoch {} completed, average training loss is {}'.format(
          epoch+1, losses.mean()))
  print('Testing Acc.: {}'.format(current_test_accuracy))
  
# Restore best model
ckpt = tf.train.get_checkpoint_state('one_layer_best_fc_retrain')
saver.restore(sess, ckpt.model_checkpoint_path)
_ = test_accuracy_bin_rate_fc(bin_rate_feed, verbose=True)

test_accuracy_bin_rate_fc(bin_rate_feed)<jupyter_output><empty_output><jupyter_text>3.4 Extract conv weights<jupyter_code>with open('NP_WEIGHTS_three_layers_bin_reg_lrelu25.pck', 'wb') as f:
  pickle.dump((w1_values, b1_values, w2_1_values, b2_1_values, w2_2_values, b2_2_values, w2_3_values, b2_3_values, w3_1_values, w3_2_values, w3_3_values), f)
w3_1_values<jupyter_output><empty_output><jupyter_text>4 Add 3 depth separable conv layers  4.1 Network definition<jupyter_code>def lrelu(x):
    return tf.maximum(x * 0.25, x)
# Define the network
def net4(input, thresh_ph, bin_rate_ph):
  # Convolution
  #conv = slim.conv2d(input, 3, [3, 3], rate=1, activation_fn=tf.nn.relu,
  #                   padding='SAME', scope='conv1')
  conv = slim.conv2d(input, 3, [3, 3], rate=1, activation_fn=lrelu,
                     padding='SAME', scope='conv1')
  
  # One depth separable conv
  conv_2_1 = slim.conv2d(conv[...,:1], 1, [3, 3], rate=1, activation_fn=lrelu,
                     padding='SAME', scope='conv2_1')
  conv_3_1 = slim.conv2d(conv_2_1, 1, [3, 3], rate=1, activation_fn=lrelu,
                     padding='SAME', scope='conv3_1')
  conv_4_1 = slim.conv2d(conv_3_1, 1, [3, 3], rate=1, activation_fn=None,
                     padding='SAME', scope='conv4_1')
  
  conv_2_2 = slim.conv2d(conv[...,1:2], 1, [3, 3], rate=1, activation_fn=lrelu,
                     padding='SAME', scope='conv2_2')
  conv_3_2 = slim.conv2d(conv_2_2, 1, [3, 3], rate=1, activation_fn=lrelu,
                     padding='SAME', scope='conv3_2')
  conv_4_2 = slim.conv2d(conv_3_2, 1, [3, 3], rate=1, activation_fn=None,
                     padding='SAME', scope='conv4_2')
  
  conv_2_3 = slim.conv2d(conv[...,2:3], 1, [3, 3], rate=1, activation_fn=lrelu,
                     padding='SAME', scope='conv2_3')
  conv_3_3 = slim.conv2d(conv_2_3, 1, [3, 3], rate=1, activation_fn=lrelu,
                     padding='SAME', scope='conv3_3')
  conv_4_3 = slim.conv2d(conv_3_3, 1, [3, 3], rate=1, activation_fn=None,
                     padding='SAME', scope='conv4_3')
  
  
  conv3 = tf.concat([conv_3_1, conv_3_2, conv_3_3], axis=-1)
  
  # Output thresholding
  conv_bin = binarize_tensor_differentiable(conv3, thresh_ph, bin_rate_ph)
  
  # Custom pooling (central division + sum overlapping)
  pool = custom_pooling(conv_bin)
  
  # Flatten + dense
  flat = tf.layers.flatten(pool)
  dense = tf.layers.dense(flat, 50, name='dense1', activation=tf.nn.relu)
  out = tf.layers.dense(dense, 10, name='dense2')
  
  return out
# Define the graph
tf.reset_default_graph()

in_image_ph = tf.placeholder(tf.float32, [BATCH_SIZE,28,28,1])
gt_label_ph = tf.placeholder(tf.uint8)
thresh = tf.Variable(tf.random.normal([3]), name='out_thresholds')
bin_rate_ph = tf.placeholder(tf.float32, ())

out_label_op = net4(in_image_ph, thresh, bin_rate_ph)

pred_op = tf.dtypes.cast(
            tf.keras.backend.argmax(out_label_op),
            tf.uint8)

loss_op = tf.reduce_mean(
          tf.keras.backend.sparse_categorical_crossentropy(gt_label_ph,
                                                           out_label_op,
                                                           from_logits=True))

acc_op = tf.contrib.metrics.accuracy(gt_label_ph, pred_op)
# Create session
sess = tf.Session()
sess.run(tf.global_variables_initializer())
#[n.name for n in tf.get_default_graph().as_graph_def().node]
# Define regulariser
def customRegularizerConv(x):
  return tf.math.cos(2/ROUNDING_STEP_CONV*np.pi*(x-ROUNDING_STEP_CONV/2))+1.

def customRegularizerBias(x):
  return tf.math.cos(2/ROUNDING_STEP_BIAS*np.pi*(x-ROUNDING_STEP_BIAS/2))+1.

with tf.variable_scope('conv1', reuse=True) as scope_conv:
  w1 = tf.get_variable('weights')
  b1 = tf.get_variable('biases')
  
with tf.variable_scope('conv2_1', reuse=True) as scope_conv:
  w2_1 = tf.get_variable('weights')
  b2_1 = tf.get_variable('biases')
with tf.variable_scope('conv2_2', reuse=True) as scope_conv:
  w2_2 = tf.get_variable('weights')
  b2_2 = tf.get_variable('biases')
with tf.variable_scope('conv2_3', reuse=True) as scope_conv:
  w2_3 = tf.get_variable('weights')
  b2_3 = tf.get_variable('biases')
  
with tf.variable_scope('conv3_1', reuse=True) as scope_conv:
  w3_1 = tf.get_variable('weights')
  b3_1 = tf.get_variable('biases')
with tf.variable_scope('conv3_2', reuse=True) as scope_conv:
  w3_2 = tf.get_variable('weights')
  b3_2 = tf.get_variable('biases')
with tf.variable_scope('conv3_3', reuse=True) as scope_conv:
  w3_3 = tf.get_variable('weights')
  b3_3 = tf.get_variable('biases')
  
with tf.variable_scope('conv4_1', reuse=True) as scope_conv:
  w4_1 = tf.get_variable('weights')
with tf.variable_scope('conv4_2', reuse=True) as scope_conv:
  w4_2 = tf.get_variable('weights')
with tf.variable_scope('conv4_3', reuse=True) as scope_conv:
  w4_3 = tf.get_variable('weights')

NUM_PARAM_REG = np.prod(([int(e) for e in w1.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in b1.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in w2_1.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in b2_1.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in w2_2.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in b2_2.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in w2_3.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in b2_3.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in w3_1.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in b3_1.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in w3_2.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in b3_2.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in w3_3.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in b3_3.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in w4_1.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in w4_2.shape]))
NUM_PARAM_REG += np.prod(([int(e) for e in w4_3.shape]))

reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w1))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerBias(b1))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w2_1))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerBias(b2_1))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w2_2))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerBias(b2_2))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w2_3))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerBias(b2_3))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w3_1))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerBias(b3_1))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w3_2))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerBias(b3_2))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w3_3))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerBias(b3_3))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w4_1))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w4_2))
reg_losses = 27./NUM_PARAM_REG *tf.reduce_mean(customRegularizerConv(w4_3))

reg_factor_ph = tf.placeholder(tf.float32)
loss_with_reg_op = loss_op + reg_factor_ph * reg_losses

lr_ph = tf.placeholder(tf.float32)
opt = tf.train.AdamOptimizer(learning_rate=lr_ph, name='Adam_reg')
opt_op = opt.minimize(loss_with_reg_op)
sess.run(tf.variables_initializer(opt.variables()))<jupyter_output>W0727 09:15:30.544577 139675524298624 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where<jupyter_text>4.2 Training<jupyter_code>"""
# Reg then bin
for epoch in range(EPOCHS*12):
  if epoch < EPOCHS * 4:
    lr_feed = LR
    reg_factor_feed = 0.
    bin_rate_feed = 1.
  elif epoch < EPOCHS * 8:
    lr_feed = LR / 2.
    reg_factor_feed = REG_CONSTANT
    bin_rate_feed = 1.
  else:
    lr_feed = LR / 4.
    reg_factor_feed = REG_CONSTANT
    bin_rate_feed = MAX_BIN_RATE + 1
    
# Bin then reg
for epoch in range(EPOCHS*12):
  if epoch < EPOCHS * 4:
    lr_feed = LR
    reg_factor_feed = 0.
    bin_rate_feed = 1.
  elif epoch < EPOCHS * 8:
    lr_feed = LR / 2.
    reg_factor_feed = 0.
    bin_rate_feed = MAX_BIN_RATE + 1
  else:
    lr_feed = LR / 4.
    reg_factor_feed = REG_CONSTANT
    bin_rate_feed = MAX_BIN_RATE + 1
"""
# Bin then reg
for epoch in range(EPOCHS*12):
  if epoch < EPOCHS * 4:
    lr_feed = LR
    reg_factor_feed = 0.
    bin_rate_feed = 1.
  elif epoch < EPOCHS * 8:
    lr_feed = LR / 2.
    reg_factor_feed = 0.
    bin_rate_feed = MAX_BIN_RATE + 1
  else:
    lr_feed = LR / 4.
    reg_factor_feed = REG_CONSTANT
    bin_rate_feed = MAX_BIN_RATE + 1
    
  random_perm = np.random.permutation(x_train.shape[0])
  losses = np.zeros(x_train.shape[0] // BATCH_SIZE)
  for i in range(x_train.shape[0] // BATCH_SIZE):
    start = i * BATCH_SIZE
    stop = start + BATCH_SIZE
    selected = random_perm[start:stop]
    
    xs = np.expand_dims(x_train[selected],-1)
    ys = y_train[selected]
    
    _, current_loss = sess.run([opt_op, loss_op],
                       feed_dict={in_image_ph: xs,
                                  gt_label_ph: ys,
                                  lr_ph: lr_feed,
                                  reg_factor_ph: reg_factor_feed,
                                  bin_rate_ph: bin_rate_feed})

    losses[i] = current_loss
  
  print('Epoch {} completed, average training loss is {}'.format(
          epoch+1, losses.mean()))
  test_accuracy_bin_rate(bin_rate_feed)
  
test_accuracy_bin_rate(bin_rate_feed)
# Save models, for not having to fully retrain it each time.

# Testing Acc.: 0.9714000046253204 for bin reg
# 0.9679000043869018 for reg bin
saver = tf.train.Saver()
saver.save(sess, 'four_layers_bin_reg_lrelu025_bis/model.ckpt')
!zip -r four_layers_bin_reg_lrelu025_bis.zip four_layers_bin_reg_lrelu025_bis<jupyter_output>adding: four_layers_bin_reg_lrelu025_bis/ (stored 0%)
  adding: four_layers_bin_reg_lrelu025_bis/model.ckpt.data-00000-of-00001 (deflated 11%)
  adding: four_layers_bin_reg_lrelu025_bis/checkpoint (deflated 42%)
  adding: four_layers_bin_reg_lrelu025_bis/model.ckpt.meta (deflated 90%)
  adding: four_layers_bin_reg_lrelu025_bis/model.ckpt.index (deflated 58%)<jupyter_text>4.3 Re-training FC after rounding the conv kernels  4.3.1 Reload Network<jupyter_code>!unzip four_layers_bin_reg_lrelu025.zip
# Define the graph
tf.reset_default_graph()

in_image_ph = tf.placeholder(tf.float32, [BATCH_SIZE_FC,28,28,1])
gt_label_ph = tf.placeholder(tf.uint8)
thresh = tf.Variable(tf.random.normal([3]), name='out_thresholds')
bin_rate_ph = tf.placeholder(tf.float32, ())

out_label_op = net4(in_image_ph, thresh, bin_rate_ph)

pred_op = tf.dtypes.cast(
            tf.keras.backend.argmax(out_label_op),
            tf.uint8)

loss_op = tf.reduce_mean(
          tf.keras.backend.sparse_categorical_crossentropy(gt_label_ph,
                                                           out_label_op,
                                                           from_logits=True))

acc_op = tf.contrib.metrics.accuracy(gt_label_ph, pred_op)

saver = tf.train.Saver()
sess = tf.Session()
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state('four_layers_bin_reg_lrelu025_bis')
saver.restore(sess, ckpt.model_checkpoint_path)
test_accuracy_bin_rate_fc(MAX_BIN_RATE)<jupyter_output>W0727 10:30:07.158785 139675524298624 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file APIs to check for files with this prefix.<jupyter_text>4.3.2 Round weights<jupyter_code>with tf.variable_scope('conv1', reuse=True) as scope_conv:
  w1 = tf.get_variable('weights')
  b1 = tf.get_variable('biases')
  
with tf.variable_scope('conv2_1', reuse=True) as scope_conv:
  w2_1 = tf.get_variable('weights')
  b2_1 = tf.get_variable('biases')
with tf.variable_scope('conv2_2', reuse=True) as scope_conv:
  w2_2 = tf.get_variable('weights')
  b2_2 = tf.get_variable('biases')
with tf.variable_scope('conv2_3', reuse=True) as scope_conv:
  w2_3 = tf.get_variable('weights')
  b2_3 = tf.get_variable('biases')
  
with tf.variable_scope('conv3_1', reuse=True) as scope_conv:
  w3_1 = tf.get_variable('weights')
  b3_1 = tf.get_variable('biases')
with tf.variable_scope('conv3_2', reuse=True) as scope_conv:
  w3_2 = tf.get_variable('weights')
  b3_2 = tf.get_variable('biases')
with tf.variable_scope('conv3_3', reuse=True) as scope_conv:
  w3_3 = tf.get_variable('weights')
  b3_3 = tf.get_variable('biases')
  
with tf.variable_scope('conv4_1', reuse=True) as scope_conv:
  w4_1 = tf.get_variable('weights')
with tf.variable_scope('conv4_2', reuse=True) as scope_conv:
  w4_2 = tf.get_variable('weights')
with tf.variable_scope('conv4_3', reuse=True) as scope_conv:
  w4_3 = tf.get_variable('weights')
# Rounding operation
rounding_weights1_op = tf.assign(w1, 
                          tf.round(w1/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)
rounding_bias1_op = tf.assign(b1, 
                          tf.round(b1/ROUNDING_STEP_BIAS)*ROUNDING_STEP_BIAS)

rounding_weights2_1_op = tf.assign(w2_1, 
                          tf.round(w2_1/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)
rounding_bias2_1_op = tf.assign(b2_1, 
                          tf.round(b2_1/ROUNDING_STEP_BIAS)*ROUNDING_STEP_BIAS)
rounding_weights2_2_op = tf.assign(w2_2, 
                          tf.round(w2_2/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)
rounding_bias2_2_op = tf.assign(b2_2, 
                          tf.round(b2_2/ROUNDING_STEP_BIAS)*ROUNDING_STEP_BIAS)
rounding_weights2_3_op = tf.assign(w2_3, 
                          tf.round(w2_3/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)
rounding_bias2_3_op = tf.assign(b2_3, 
                          tf.round(b2_3/ROUNDING_STEP_BIAS)*ROUNDING_STEP_BIAS)

rounding_weights3_1_op = tf.assign(w3_1, 
                          tf.round(w3_1/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)
rounding_bias3_1_op = tf.assign(b3_1, 
                          tf.round(b3_1/ROUNDING_STEP_BIAS)*ROUNDING_STEP_BIAS)
rounding_weights3_2_op = tf.assign(w3_2, 
                          tf.round(w3_2/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)
rounding_bias3_2_op = tf.assign(b3_2, 
                          tf.round(b3_2/ROUNDING_STEP_BIAS)*ROUNDING_STEP_BIAS)
rounding_weights3_3_op = tf.assign(w3_3, 
                          tf.round(w3_3/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)
rounding_bias3_3_op = tf.assign(b3_3, 
                          tf.round(b3_3/ROUNDING_STEP_BIAS)*ROUNDING_STEP_BIAS)

rounding_weights4_1_op = tf.assign(w4_1, 
                          tf.round(w4_1/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)
rounding_weights4_2_op = tf.assign(w4_2, 
                          tf.round(w4_2/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)
rounding_weights4_3_op = tf.assign(w4_3, 
                          tf.round(w4_3/ROUNDING_STEP_CONV)*ROUNDING_STEP_CONV)

_ = sess.run(rounding_weights1_op)
_ = sess.run(rounding_bias1_op)

_ = sess.run(rounding_weights2_1_op)
_ = sess.run(rounding_bias2_1_op)
_ = sess.run(rounding_weights2_2_op)
_ = sess.run(rounding_bias2_2_op)
_ = sess.run(rounding_weights2_3_op)
_ = sess.run(rounding_bias2_3_op)

_ = sess.run(rounding_weights3_1_op)
_ = sess.run(rounding_bias3_1_op)
_ = sess.run(rounding_weights3_2_op)
_ = sess.run(rounding_bias3_2_op)
_ = sess.run(rounding_weights3_3_op)
_ = sess.run(rounding_bias3_3_op)

_ = sess.run(rounding_weights4_1_op)
_ = sess.run(rounding_weights4_2_op)
_ = sess.run(rounding_weights4_3_op)

test_accuracy_bin_rate_fc(MAX_BIN_RATE, verbose=True)

# Show final distribution of weights
w1_values = sess.run(w1)
b1_values = sess.run(b1)

w2_1_values = sess.run(w2_1)
b2_1_values = sess.run(b2_1)
w2_2_values = sess.run(w2_2)
b2_2_values = sess.run(b2_2)
w2_3_values = sess.run(w2_3)
b2_3_values = sess.run(b2_3)

w3_1_values = sess.run(w3_1)
b3_1_values = sess.run(b3_1)
w3_2_values = sess.run(w3_2)
b3_2_values = sess.run(b3_2)
w3_3_values = sess.run(w3_3)
b3_3_values = sess.run(b3_3)

w4_1_values = sess.run(w4_1)
w4_2_values = sess.run(w4_2)
w4_3_values = sess.run(w4_3)

kernel_values = (list(w1_values.flatten()) + 
                  list(b1_values.flatten()) +
                  list(w2_1_values.flatten()) +
                  list(b2_1_values.flatten()) +
                  list(w2_2_values.flatten()) +
                  list(b2_2_values.flatten()) +
                  list(w2_3_values.flatten()) +
                  list(b2_3_values.flatten()) +
                  list(w3_1_values.flatten()) +
                  list(b3_1_values.flatten()) +
                  list(w3_2_values.flatten()) +
                  list(b3_2_values.flatten()) +
                  list(w3_3_values.flatten()) + 
                  list(b3_3_values.flatten()) +
                  list(w4_1_values.flatten()) +
                  list(w4_2_values.flatten()) +
                  list(w4_3_values.flatten()))
fig = plt.figure()
plt.scatter(kernel_values, [1]*len(kernel_values))<jupyter_output>Testing Acc.: 0.9413999971747399<jupyter_text>4.3.3 Retrain FC only<jupyter_code>with tf.variable_scope('dense1', reuse=True) as scope_conv:
  fc1_k = tf.get_variable('kernel')
  fc1_b = tf.get_variable('bias')

with tf.variable_scope('dense2', reuse=True) as scope_conv:
  fc2_k = tf.get_variable('kernel')
  fc2_b = tf.get_variable('bias')

lr_ph = tf.placeholder(tf.float32)
opt_fc = tf.train.AdamOptimizer(learning_rate=lr_ph, name='Adam_reg')
opt_fc_op = opt_fc.minimize(loss_op, var_list=[fc1_k, fc1_b,
                                               fc2_k, fc2_b])

sess.run(tf.variables_initializer(opt_fc.variables()))
# Very long training:
bin_rate_feed = MAX_BIN_RATE + 1
max_test_accuracy = 0.
for epoch in range(EPOCHS_FC*10):
  if epoch < EPOCHS_FC:
    lr_feed = LR_FC*8
  elif epoch < 3*EPOCHS_FC:
    lr_feed = LR_FC * 4
  elif epoch < 6*EPOCHS_FC:
    lr_feed = LR_FC * 2
  elif epoch < 8:
    lr_feed = LR_FC
  elif epoch < 9:
    lr_feed = LR_FC / 2.
  else:
    lr_feed = LR_FC / 5.
  random_perm = np.random.permutation(x_train.shape[0])
  losses = np.zeros(x_train.shape[0] // BATCH_SIZE_FC)
  for i in range(x_train.shape[0] // BATCH_SIZE_FC):
    start = i * BATCH_SIZE_FC
    stop = start + BATCH_SIZE_FC
    selected = random_perm[start:stop]
    
    xs = np.expand_dims(x_train[selected],-1)
    ys = y_train[selected]
    
    _, current_loss = sess.run([opt_fc_op, loss_op],
                       feed_dict={in_image_ph: xs,
                                  gt_label_ph: ys,
                                  bin_rate_ph: bin_rate_feed,
                                  lr_ph: lr_feed})

    losses[i] = current_loss
  
  current_test_accuracy = test_accuracy_bin_rate_fc(bin_rate_feed)
  # Save best model
  if current_test_accuracy > max_test_accuracy:
    saver.save(sess, 'one_layer_best_fc_retrain/model.ckpt')
    max_test_accuracy = current_test_accuracy
  
  print('Epoch {} completed, average training loss is {}'.format(
          epoch+1, losses.mean()))
  print('Testing Acc.: {}'.format(current_test_accuracy))
  
# Restore best model
ckpt = tf.train.get_checkpoint_state('one_layer_best_fc_retrain')
saver.restore(sess, ckpt.model_checkpoint_path)
_ = test_accuracy_bin_rate_fc(bin_rate_feed, verbose=True)

test_accuracy_bin_rate_fc(bin_rate_feed)<jupyter_output><empty_output><jupyter_text>4.4 Extract conv weights<jupyter_code>with open('NP_WEIGHTS_four_layers_bin_reg_lrelu25.pck', 'wb') as f:
  pickle.dump((w1_values, b1_values,
               w2_1_values, b2_1_values, w2_2_values, b2_2_values, w2_3_values, b2_3_values,
               w3_1_values, b3_1_values, w3_2_values, b3_2_values, w3_3_values, b3_3_values, 
               w4_1_values, w4_2_values, w4_3_values), f)
w3_1_values<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11966079367693068, 'fraction_numerical': 0.03872879960480817, 'mean_word_length': 3.5522074806985984, 'pattern_counts': {'":': 0, '<': 143, '<?xml version=': 0, '>': 103, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 72, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '27', 'dir': 'jupyter-structured-clean-dedup', 'id': '654335', 'license': 'MIT', 'n_tokens_mistral': 25009, 'n_tokens_neox': 23664, 'n_words': 3837, 'path': '4 - Depth Separable Net/1 - Training Notebook/Depth_separable_convs.ipynb', 'repo_name': 'brouwa/CNNs-on-FPSPs', 'size': '60730'}