<jupyter_start><jupyter_text>Load Data<jupyter_code>with open('SVHN_data.pickle', 'rb') as f:
    tmp = pickle.load(f)
    train_dataset = tmp['train_dataset']
    train_labels = tmp['train_labels']
    valid_dataset = tmp['valid_dataset']
    valid_labels = tmp['valid_labels']
    test_dataset = tmp['test_dataset']
    test_labels = tmp['test_labels']
    del tmp
    
print('Training set', train_dataset.shape, train_labels.shape)
print('Validation set', valid_dataset.shape, valid_labels.shape)
print('Test set', test_dataset.shape, test_labels.shape)<jupyter_output>('Training set', (26721, 32, 32, 1), (26721, 5))
('Validation set', (6680, 32, 32, 1), (6680, 5))
('Test set', (13068, 32, 32, 1), (13068, 5))<jupyter_text>Final Model<jupyter_code>image_size = 32
num_labels = 11  # 0-9, + blank 
num_channels = 1 # grayscale
batch_size = 64
patch_size = 5
depthC1 = 16
depthC2 = 32
depthC3 = 64
depthC4 = 96
depthFC1 = 128
kp=0.9
#beta = 0.1

graph = tf.Graph()

with graph.as_default():
    # Input data.
    with tf.name_scope('input'):
        tf_train_dataset = tf.placeholder(tf.float32, shape=(batch_size, image_size, image_size, num_channels))
        tf_train_labels = tf.placeholder(tf.int32, shape=(batch_size, 5))
    
        tf_valid_dataset = tf.constant(valid_dataset)
        tf_valid_labels = tf.constant(valid_labels)
        tf_test_dataset = tf.constant(test_dataset)
        tf_test_labels = tf.constant(test_labels)

    # Variables.
    with tf.name_scope('conv1_para'):
        W_conv1 = tf.Variable(tf.truncated_normal([patch_size, patch_size, num_channels, depthC1],stddev=0.1))
        b_conv1 = tf.Variable(tf.ones([depthC1]))
    
    with tf.name_scope('conv2_para'):
        W_conv2 = tf.Variable(tf.truncated_normal([patch_size, patch_size, depthC1, depthC2],stddev=0.1))
        b_conv2 = tf.Variable(tf.ones([depthC2]))
    
    with tf.name_scope('conv3_para'):
        W_conv3 = tf.Variable(tf.truncated_normal([patch_size, patch_size,depthC2, depthC3],stddev=0.1))
        b_conv3 = tf.Variable(tf.ones([depthC3]))
    
    with tf.name_scope('conv4_para'):
        W_conv4 = tf.Variable(tf.truncated_normal([patch_size, patch_size,depthC3, depthC4],stddev=0.1))
        b_conv4 = tf.Variable(tf.ones([depthC4]))

    with tf.name_scope('fc1_para'):
        W_fc1 = tf.Variable(tf.truncated_normal([2*2*depthC4, depthFC1],stddev=0.1))
        b_fc1 = tf.Variable(tf.ones([depthFC1]))
    
    with tf.name_scope('fc_out1_para'):
        W_o1 = tf.Variable(tf.truncated_normal([depthFC1, num_labels],stddev=0.1))
        b_o1 = tf.Variable(tf.ones([num_labels]))
   
    with tf.name_scope('fc_out2_para'):
        W_o2 = tf.Variable(tf.truncated_normal([depthFC1, num_labels],stddev=0.1))
        b_o2 = tf.Variable(tf.ones([num_labels]))
    
    with tf.name_scope('fc_out3_para'):
        W_o3 = tf.Variable(tf.truncated_normal([depthFC1, num_labels],stddev=0.1))
        b_o3 = tf.Variable(tf.ones([num_labels]))
    
    with tf.name_scope('fc_out4_para'):
        W_o4 = tf.Variable(tf.truncated_normal([depthFC1, num_labels],stddev=0.1))
        b_o4 = tf.Variable(tf.ones([num_labels]))
    
    with tf.name_scope('fc_out5_para'):
        W_o5 = tf.Variable(tf.truncated_normal([depthFC1, num_labels],stddev=0.1))
        b_o5 = tf.Variable(tf.ones([num_labels]))
  
    # CNN
    def model(data, keep_prob=1):    
        #CONV
        with tf.name_scope('layer1_CONV'):
            with tf.name_scope('conv1'):
                h_conv1 = tf.nn.conv2d(data,W_conv1, [1,1,1,1],padding='SAME') + b_conv1
            h_conv1 = tf.nn.relu(h_conv1)
            h_conv1 = tf.nn.lrn(h_conv1) 
            h_conv1 = tf.nn.max_pool(h_conv1, [1,2,2,1], [1,2,2,1], 'SAME') 
        
        with tf.name_scope('layer2_CONV'):
            with tf.name_scope('conv2'):
                h_conv2 = tf.nn.conv2d(h_conv1, W_conv2, [1,1,1,1], padding='SAME') + b_conv2 
            h_conv2 = tf.nn.relu(h_conv2)
            h_conv2 = tf.nn.lrn(h_conv2) 
            h_conv2 = tf.nn.max_pool(h_conv2, [1,2,2,1], [1,2,2,1], 'SAME')
        
        with tf.name_scope('layer3_CONV'):
            with tf.name_scope('conv3'):
                h_conv3 = tf.nn.conv2d(h_conv2, W_conv3, [1,1,1,1], padding='SAME') + b_conv3
            h_conv3 = tf.nn.relu(h_conv3)
            h_conv3 = tf.nn.lrn(h_conv3)
            h_conv3 = tf.nn.max_pool(h_conv3, [1,2,2,1], [1,2,2,1], 'SAME')
        
        with tf.name_scope('layer4_CONV'):
            with tf.name_scope('conv4'):
                h_conv4 = tf.nn.conv2d(h_conv3, W_conv4, [1,1,1,1], padding='SAME') + b_conv4
            h_conv4 = tf.nn.relu(h_conv4)
            h_conv4 = tf.nn.lrn(h_conv4)
            h_conv4 = tf.nn.max_pool(h_conv4, [1,2,2,1], [1,2,2,1], 'SAME')
        
        with tf.name_scope('dropout1'):
            h_conv4 = tf.nn.dropout(h_conv4, keep_prob)
            #Reshape
            shape = h_conv4.get_shape().as_list()
            h_conv4 = tf.reshape(h_conv4, [shape[0], shape[1] * shape[2] * shape[3]])
        
        #FC
        with tf.name_scope('layer5_FC'):
            with tf.name_scope('fc1'):
                h_fc1 = tf.matmul(h_conv4, W_fc1) + b_fc1
            h_fc1 = tf.nn.relu(h_fc1)
            
        with tf.name_scope('dropout2'):
            h_fc1 = tf.nn.dropout(h_fc1, keep_prob)

        with tf.name_scope('layer_output_logits'):
        #OUTPUT
            with tf.name_scope('logits1'):
                logits1 = tf.matmul(h_fc1, W_o1) + b_o1
            with tf.name_scope('logits2'):
                logits2 = tf.matmul(h_fc1, W_o2) + b_o2
            with tf.name_scope('logits3'):
                logits3 = tf.matmul(h_fc1, W_o3) + b_o3
            with tf.name_scope('logits4'):
                logits4 = tf.matmul(h_fc1, W_o4) + b_o4
            with tf.name_scope('logits5'):
                logits5 = tf.matmul(h_fc1, W_o5) + b_o5
        return tf.pack([logits1, logits2, logits3, logits4, logits5])
  
    # Training computation.
    logits = model(tf_train_dataset, kp) 
    
    # Define loss function.
    with tf.name_scope("loss_function") as scope:
        loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits[0], tf_train_labels[:,0])) +\
               tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits[1], tf_train_labels[:,1])) +\
               tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits[2], tf_train_labels[:,2])) +\
               tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits[3], tf_train_labels[:,3])) +\
               tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits[4], tf_train_labels[:,4]))
        # Create a summary to monitor the cost function
        tf.scalar_summary("loss_function", loss)

    train_step = tf.train.AdamOptimizer().minimize(loss)
    
    def eval_accuracy(predictions, labels):
        return tf.reduce_mean( tf.reduce_min(tf.to_float(tf.equal(tf.to_int32(predictions), labels)), axis = 1))

    with tf.name_scope('accuracy'):
        # Predictions for the mini training, validation, and test data.
        train_prediction = tf.transpose(tf.argmax(logits, axis = 2))
        valid_prediction =  tf.transpose(tf.argmax(model(tf_valid_dataset), axis = 2))
        test_prediction =  tf.transpose(tf.argmax(model(tf_test_dataset), axis = 2))

        train_accuracy = eval_accuracy(train_prediction, tf_train_labels)
        valid_accuracy = eval_accuracy(valid_prediction, tf_valid_labels)
        test_accuracy = eval_accuracy(test_prediction, tf_test_labels)
    
     # Create summaries to monitor the accuracy
    tf.scalar_summary("mini-batch accuracy", train_accuracy)
    tf.scalar_summary("validation accuracy", valid_accuracy)
        
    # Merge all summaries into a single operator
    merged_summary_op = tf.merge_all_summaries()
    
    saver = tf.train.Saver(max_to_keep = 2)<jupyter_output><empty_output><jupyter_text>Graph Visualization via Tensorboard<jupyter_code>show_graph(graph)<jupyter_output><empty_output><jupyter_text>Training and Evaluation<jupyter_code>num_steps = 20001
#num_steps = 101

with tf.Session(graph=graph) as session:
    
    tf.global_variables_initializer().run()
    summary_writer = tf.train.SummaryWriter("./logs", session.graph)
    
    for step in range(num_steps):
        offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
        batch_data = train_dataset[offset:(offset + batch_size), :, :, :]
        batch_labels = train_labels[offset:(offset + batch_size),:]
        
        feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}
        if (step % 500 ==0):
            _, l, train_acc, summary_str = session.run([train_step, loss, train_accuracy, merged_summary_op], feed_dict=feed_dict)
            summary_writer.add_summary(summary_str, step)
        else:
            _, l, train_acc = session.run([train_step, loss, train_accuracy], feed_dict=feed_dict)
           
        
        #print('step: %d' % step),
        if (step % 500 == 0):  #500
            print('Minibatch loss at step %d: %f' % (step, l))
            print('Minibatch accuracy: %.1f%%' % (train_acc*100))
            print('Validation accuracy: %.1f%%' % (valid_accuracy.eval()*100))
            print('')
            
    print('Test accuracy: %.1f%%' % (test_accuracy.eval()*100))
    save_path = saver.save(session, "./ckpt_folder/CNN_trained_finalModel.ckpt",global_step=step)
    print("Model saved in file: %s" % save_path)<jupyter_output>Minibatch loss at step 0: 12.580278
Minibatch accuracy: 0.0%
Validation accuracy: 0.1%

Minibatch loss at step 500: 5.146176
Minibatch accuracy: 7.8%
Validation accuracy: 8.7%

Minibatch loss at step 1000: 3.704948
Minibatch accuracy: 32.8%
Validation accuracy: 39.4%

Minibatch loss at step 1500: 2.200338
Minibatch accuracy: 50.0%
Validation accuracy: 52.2%

Minibatch loss at step 2000: 1.941166
Minibatch accuracy: 59.4%
Validation accuracy: 56.8%

Minibatch loss at step 2500: 1.731720
Minibatch accuracy: 56.2%
Validation accuracy: 59.9%

Minibatch loss at step 3000: 1.737435
Minibatch accuracy: 50.0%
Validation accuracy: 62.7%

Minibatch loss at step 3500: 1.293933
Minibatch accuracy: 68.8%
Validation accuracy: 61.1%

Minibatch loss at step 4000: 1.127986
Minibatch accuracy: 73.4%
Validation accuracy: 63.4%

Minibatch loss at step 4500: 1.440659
Minibatch accuracy: 65.6%
Validation accuracy: 63.1%

Minibatch loss at step 5000: 0.942206
Minibatch accuracy: 76.6%
Validation accuracy: 64[...]
{'fraction_non_alphanumeric': 0.13282065888161018, 'fraction_numerical': 0.050412987752776986, 'mean_word_length': 3.126126126126126, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '73116', 'license': 'MIT', 'n_tokens_mistral': 4128, 'n_tokens_neox': 3869, 'n_words': 660, 'path': '3_SVHN_CNN_final.ipynb', 'repo_name': 'DerrickChi/ML-svhn', 'size': '10533'}