<jupyter_start><jupyter_text>Classify images from MNIST using LeNet  DatasetDownload the [dataset](http://deeplearning.net/data/mnist/mnist.pkl.gz) to your workspace (i.e. the notebook folder).<jupyter_code>import cPickle, gzip

# Load the dataset
f = gzip.open('mnist.pkl.gz', 'rb')
train_set, valid_set, _ = cPickle.load(f)
f.close()
print train_set[0].shape, train_set[1].shape
print valid_set[0].shape, valid_set[1].shape
import numpy as np
train_x = np.reshape(train_set[0], (50000, 1, 28, 28)).astype(np.float32, copy=False)
train_y = np.array(train_set[1]).astype(np.int32, copy=False)
valid_x = np.reshape(valid_set[0], (10000, 1, 28, 28))
%matplotlib inline
import matplotlib.pyplot as plt
plt.imshow(train_x[0][0])<jupyter_output><empty_output><jupyter_text>Create the CNN modelTODO: plot the net structure<jupyter_code>from singa import net as ffnet
from singa.layer import Conv2D, MaxPooling2D, Dropout, Activation, Flatten, Dense
from singa import optimizer, loss, metric
from singa import layer
layer.engine = 'singacpp'
net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())
net.add(Conv2D('conv1', 32, 3, 2, input_sample_shape=(1,28,28)))
net.add(Activation('relu1'))
net.add(Conv2D('conv2', 32, 3, 2))
net.add(Activation('relu2'))
net.add(MaxPooling2D('pool', 3, 2))
net.add(Flatten('flat'))
net.add(Dense('dense', 10))<jupyter_output>conv1 (32, 14, 14)
relu1 (32, 14, 14)
conv2 (32, 7, 7)
relu2 (32, 7, 7)
pool (32, 4, 4)
flat (512,)
dense (10,)<jupyter_text>Initialize the parameters* weight matrix - guassian distribution* bias - 0<jupyter_code>for pname, pval in zip(net.param_names(), net.param_values()):
    if len(pval.shape) > 1:
        pval.gaussian(0, 0.1)
    else:
        pval.set_value(0)
    print pname, pval.shape, pval.l1()<jupyter_output>conv1_weight (32, 9) 0.0764843672514
conv1_bias (32,) 0.0
conv2_weight (32, 288) 0.0803024768829
conv2_bias (32,) 0.0
dense_weight (512, 10) 0.0795410946012
dense_bias (10,) 0.0<jupyter_text>Set up the optimizer and tensors<jupyter_code>from singa import tensor
#from singa.proto import core_pb2
from singa import device
from singa import utils
cpu = device.get_default_device()

opt = optimizer.SGD(momentum=0.9, weight_decay=1e-4)
batch_size = 32
num_train_batch = train_x.shape[0] / batch_size

tx = tensor.Tensor((batch_size, 1, 28, 28))
ty = tensor.Tensor((batch_size,), cpu , tensor.int32)

# for progress bar
from tqdm import tnrange
idx = np.arange(train_x.shape[0], dtype=np.int32)<jupyter_output><empty_output><jupyter_text>Conduct SGD1. process the training data multile time, each time is called on epoch; 2. for each epoch, read the data as mini-batches in random order3. for each mini-batch, do BP and update the parameters<jupyter_code>for epoch in range(2):
    np.random.shuffle(idx)
    loss, acc = 0.0, 0.0
    
    bar = tnrange(num_train_batch, desc='Epoch %d' % epoch)
    for b in bar:
        x = train_x[idx[b * batch_size: (b + 1) * batch_size]]
        y = train_y[idx[b * batch_size: (b + 1) * batch_size]]
        tx.copy_from_numpy(x)
        ty.copy_from_numpy(y)
        grads, (l, a) = net.train(tx, ty)
        loss += l
        acc += a
        for (s, p, g) in zip(net.param_names(), net.param_values(), grads):
            opt.apply_with_lr(epoch, 0.01, g, p, str(s), b)
        # update progress bar
        bar.set_postfix(train_loss=l, train_accuracy=a)
    print 'Epoch = %d, training loss = %f, training accuracy = %f' % (epoch, loss / num_train_batch, acc / num_train_batch)<jupyter_output>Epoch = 0, training loss = 0.292792, training accuracy = 0.906530

Epoch = 1, training loss = 0.109958, training accuracy = 0.965589<jupyter_text>Save model to disk<jupyter_code>net.save('checkpoint')<jupyter_output><empty_output><jupyter_text>Load model from disk<jupyter_code>for pval in net.param_values():
    pval.set_value(0)
net.load('checkpoint')<jupyter_output>NOTE: If your model was saved using pickle, then set use_pickle=True for loading it
conv2_bias
conv2_weight
dense_weight
conv1_bias
dense_bias
conv1_weight<jupyter_text>Do prediction<jupyter_code>from PIL import Image
img = Image.open('static/digit.jpg').convert('L')
img = img.resize((28,28))
img = np.array(img, dtype=np.float32)/255
img = tensor.from_numpy(img)
img.reshape((1,1,28,28))
y=net.predict(img)
prob=tensor.to_numpy(y)[0]
plt.plot(range(10), prob)<jupyter_output><empty_output><jupyter_text>DebugPrint l1 norm or parameter and layer feature1. parameter initialization2. learning rate3. weight decay<jupyter_code>np.random.shuffle(idx)
ffnet.verbose=True
for pname, pval in zip(net.param_names(), net.param_values()):
    if len(pval.shape) > 1:
        pval.gaussian(0, 10)
    else:
        pval.set_value(0)
    print pname, pval.shape, pval.l1()
for b in range(10):
    print "\n\nEpoch %d" % b
    x = train_x[idx[b * batch_size: (b + 1) * batch_size]]
    y = train_y[idx[b * batch_size: (b + 1) * batch_size]]
    tx.copy_from_numpy(x)
    ty.copy_from_numpy(y)
    grads, (l, a) = net.train(tx, ty)
    print '\n loss = %f, params' % l
    for (s, p, g) in zip(net.param_names(), net.param_values(), grads):
        opt.apply_with_lr(epoch, 0.01, g, p, str(s), b)
        print s, p.l1()

def vis_square(data):
    """Take an array of shape (n, height, width) or (n, height, width, 3)
       and visualize each (height, width) thing in a grid of size approx. sqrt(n) by sqrt(n)"""
    
    # normalize data for display
    data = (data - data.min()) / (data.max() - data.min())
    
    # force the number of filters to be square
    n = int(np.ceil(np.sqrt(data.shape[0])))
    padding = (((0, n ** 2 - data.shape[0]),
               (0, 1), (0, 1))                 # add some space between filters
               + ((0, 0),) * (data.ndim - 3))  # don't pad the last dimension (if there is one)
    data = np.pad(data, padding, mode='constant', constant_values=1)  # pad with ones (white)
    
    # tile the filters into an image
    data = data.reshape((n, n) + data.shape[1:]).transpose((0, 2, 1, 3) + tuple(range(4, data.ndim + 1)))
    data = data.reshape((n * data.shape[1], n * data.shape[3]) + data.shape[4:])
    
    plt.imshow(data); plt.axis('off')
np.random.shuffle(idx)
ffnet.verbose=False
net.load('checkpoint')
b=1
x = train_x[idx[b * batch_size: (b + 1) * batch_size]]    
tx.copy_from_numpy(x)

r = net.forward(False, tx, ['relu1', 'relu2'])


    
r1 = tensor.to_numpy(r['relu1'])[0]
vis_square(r1)
r2 = tensor.to_numpy(r['relu2'])[0]
vis_square(r2)
p=net.param_values()[2]
print p.shape
vis_square(tensor.to_numpy(p)[0].reshape(32, 3,3))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14817617678220069, 'fraction_numerical': 0.047373997275616773, 'mean_word_length': 4.2864, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '207415', 'license': 'Apache-2.0', 'n_tokens_mistral': 2752, 'n_tokens_neox': 2561, 'n_words': 617, 'path': 'doc/en/docs/notebook/cnn.ipynb', 'repo_name': 'pavan87/Machine-learning', 'size': '6607'}