<jupyter_start><jupyter_text>Assignement 2 Machine Learning 159738 -  Combine Convolutional Neural Network(CNN) and Support Vector Machine (SVM) classification for Cifar 10 dataset  Name: <NAME> ID: 18042783 Name: <NAME>    ID: 19039281  Reference:     * https://arxiv.org/pdf/1712.03541v2.pdf    * https://github.com/AFAgarap/cnn-svm  Datasets: Cifar 10* https://www.cs.toronto.edu/~kriz/cifar.html* https://keras.io/examples/cifar10_cnn/<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import matplotlib as mpl
import seaborn as sns
import numpy as np
import patsy
import os
#from plt import Image 
import cv2
%matplotlib inline
# TensorFlow and tf.keras
import tensorflow as tf
from tensorflow import keras

# Helper libraries
import numpy as np
import matplotlib.pyplot as plt

import time
import sys
print(tf.__version__)
from pylab import rcParams

sns.set(style="ticks")
#sns.set_style("whitegrid")
rcParams['figure.dpi'] = 72
rcParams['lines.linewidth'] = 2
rcParams['axes.facecolor'] = 'white'
rcParams['patch.edgecolor'] = 'white'
rcParams['font.family'] = 'StixGeneral'
rcParams['figure.figsize'] = 10,8
rcParams['font.size'] = 14
rcParams['axes.labelsize'] = 'large'
rcParams['xtick.labelsize'] = 14
rcParams['ytick.labelsize'] = 14

pd.options.display.max_columns = 50

import keras
from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D
from google.colab import drive
drive.mount('/content/drive/')
!ls "/content/drive/My Drive/Colab Notebooks/model"
save_dir = os.path.join(os.getcwd(), 'saved_models')
model_name = 'keras_cifar10_trained_model.h5'
save_dir<jupyter_output><empty_output><jupyter_text>New Section<jupyter_code># The data, split between train and test sets:
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
def load_label_names():
    return ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
class_names = load_label_names()
num_classes = 10
# plot first few images
for i in range(9):
    plt.subplot(330 + 1 + i)
    # define subplot
    plt.imshow(x_train[i])
# show the figure
plt.show()
# Convert class vectors to binary class matrices.
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)<jupyter_output><empty_output><jupyter_text>RGB  input ceshi<jupyter_code>x_train.shape
x_train = x_train/255
x_test = x_test/255
#print(x_train[0])
#training_data = []
#for i in x_train:
#  X_train_list = cv2.cvtColor(i, cv2.COLOR_BGR2GRAY)
#  training_data.append(X_train_list)
  
  
#for i in x_train:
#  print(i.shape)
#  X_train_list = cv2.cvtColor(i, cv2.COLOR_BGR2GRAY)
#  training_data.append([X_train_list])
  
#trainfeaturex = np.array(training_data).reshape(-1, 32, 32, 1)
#print(trainfeaturex.shape)<jupyter_output><empty_output><jupyter_text>Create CNN model Initializes the CNN-Softmax model    * param alpha: The learning rate to be used by the model.    * param batch_size: The number of batches to use for training/validation/testing.    * param num_classes: The number of classes in the dataset.    * param num_features: The number of features in the dataset.    * param penalty_parameter: The SVM C penalty parameter.<jupyter_code>batch_size = 100
num_classes = 10
epochs = 1000
data_augmentation = True
num_predictions = 20
num_classes = y_train.shape[1]
#num_features is one image size shape
num_features = x_train[1].shape
alpha=1e-3
penalty_parameter = 0.5 #this parameter should check again
num_features 
def weight_variable(shape):
    """Returns a weight matrix consisting of arbitrary values.

    :param shape: The shape of the weight matrix to create.
    :return: The weight matrix consisting of arbitrary values.
     """
    initial = tf.truncated_normal(shape, stddev=0.1)
    return tf.Variable(initial)

def bias_variable(shape):
    """Returns a bias matrix consisting of 0.1 values.

    :param shape: The shape of the bias matrix to create.
    :return: The bias matrix consisting of 0.1 values.
    """
    initial = tf.constant(0.1, shape=shape)
    return tf.Variable(initial)

def conv2d(features, weight):
    """Produces a convolutional layer that filters an image subregion

    :param features: The layer input.
    :param weight: The size of the layer filter.
    :return: Returns a convolutional layer.
    """
    return tf.nn.conv2d(features, weight, strides=[1, 1, 1, 1], padding='SAME')

def max_pool_2x2(features):
    """Downnsamples the image based on convolutional layer

    :param features: The input to downsample.
    :return: Downsampled input.
    """
    return tf.nn.max_pool(features, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')

with tf.name_scope('input'):
    # [BATCH_SIZE, NUM_FEATURES]
  #  x_input = tf.placeholder(dtype=tf.float32, shape=[None, num_features], name='x_input')
    x_input = tf.placeholder(dtype=tf.float32, shape=[None, 32,32,3], name='x_input')
    # [BATCH_SIZE, NUM_CLASSES]
    y_input = tf.placeholder(dtype=tf.float32, shape=[None, num_classes], name='actual_label')     
print("X-imput", x_input,"Y-imput", y_input)
print(type(y_input))
# First convolutional layer
first_conv_weight = weight_variable([5, 5, 3, 32])
first_conv_bias = bias_variable([32])

input_image = tf.reshape(x_input, [-1, 32, 32, 3])

first_conv_activation = tf.nn.relu(conv2d(input_image, first_conv_weight) + first_conv_bias)
first_conv_pool = max_pool_2x2(first_conv_activation)

# Second convolutional layer
second_conv_weight = weight_variable([5, 5, 32, 64])
second_conv_bias = bias_variable([64])

second_conv_activation = tf.nn.relu(conv2d(first_conv_pool, second_conv_weight) + second_conv_bias)
second_conv_pool = max_pool_2x2(second_conv_activation)

# Fully-connected layer (Dense Layer)
dense_layer_weight = weight_variable([8 * 8 * 64, 1024])
dense_layer_bias = bias_variable([1024])

second_conv_pool_flatten = tf.reshape(second_conv_pool, [-1, 8 * 8 * 64])
dense_layer_activation = tf.nn.relu(tf.matmul(second_conv_pool_flatten, dense_layer_weight) + dense_layer_bias)

# Dropout, to avoid over-fitting
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(dense_layer_activation, keep_prob)

# Readout layer
readout_weight = weight_variable([1024, num_classes])
readout_bias = bias_variable([num_classes])

output = tf.matmul(h_fc1_drop, readout_weight) + readout_bias        
with tf.name_scope('svm'):
    regularization_loss = tf.reduce_mean(tf.square(readout_weight))
    hinge_loss = tf.reduce_mean(
        tf.square(tf.maximum(tf.zeros([batch_size, num_classes]), 1 - y_input * output)))
    with tf.name_scope('loss'):
        loss = regularization_loss + penalty_parameter * hinge_loss
    tf.summary.scalar('loss', loss)

    optimizer = tf.train.AdamOptimizer(learning_rate=alpha).minimize(loss)

with tf.name_scope('accuracy'):
    output = tf.identity(tf.sign(output), name='prediction')
    correct_prediction = tf.equal(tf.argmax(output, 1), tf.argmax(y_input, 1))
    with tf.name_scope('accuracy'):
       accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    tf.summary.scalar('accuracy', accuracy)

    merged = tf.summary.merge_all()

    x_input_new = x_input
    y_input_new = y_input
    keep_prob_new = keep_prob
    output_new = output
    loss_new = loss
    optimizer_new = optimizer
    accuracy_new = accuracy
    merged_new = merged
# 此处为google drive中的文件路径,drive为之前指定的工作根目录，要加上
os.chdir("/content/drive/My Drive/Colab Notebooks/model") 
os.getcwd()
file_path = os.getcwd()
log_path  = file_path
checkpoint_path = file_path
train_data = y_train
test_data = y_test
def next_batch(num, data, labels):

  idx = np.arange(0 , len(data))
  np.random.shuffle(idx)
  idx = idx[:num]
  data_shuffle = [data[ i] for i in idx]
  labels_shuffle = [labels[ i] for i in idx]
  #return data_shuffle, labels_shuffle
  return np.asarray(data_shuffle), np.asarray(labels_shuffle)
if not os.path.exists(path=log_path):
    os.mkdir(log_path)

if not os.path.exists(path=checkpoint_path):
    os.mkdir(checkpoint_path)

saver = tf.train.Saver(max_to_keep=4)

init = tf.global_variables_initializer()

timestamp = str(time.asctime())

train_writer = tf.summary.FileWriter(logdir=log_path + timestamp + '-training', graph=tf.get_default_graph())
epochs = 1000
with tf.Session() as sess:
    sess.run(init)

    checkpoint = tf.train.get_checkpoint_state(checkpoint_path)

    if checkpoint and checkpoint.model_checkpoint_path:
        saver = tf.train.import_meta_graph(checkpoint.model_checkpoint_path + '.meta')
        saver.restore(sess, tf.train.latest_checkpoint(checkpoint_path))

    for index in range(epochs):
        # train by batch
        #batch_features, batch_labels = train_data.next_batch(self.batch_size)
        
        batch_features, batch_labels = next_batch(batch_size, x_train, y_train )
        #print(batch_features.shape)
        #print(batch_labels.shape)  
        batch_labels[batch_labels == 0] = -1

        # input dictionary with dropout of 50%
        feed_dict = {x_input: batch_features, y_input: batch_labels, keep_prob: 0.5}

        # run the train op
        summary, _, loss = sess.run([merged_new, optimizer_new, loss_new], feed_dict=feed_dict)

        # every 100th step and at 0,
        if index % 100 == 0:
            feed_dict = {x_input: batch_features, y_input: batch_labels, keep_prob: 1.0}

            # get the accuracy of training
            train_accuracy = sess.run(accuracy_new, feed_dict=feed_dict)

            # display the training accuracy
            print('step: {}, training accuracy : {}, training loss : {}'.format(index, train_accuracy, loss))

            train_writer.add_summary(summary=summary, global_step=index)

            saver.save(sess, save_path=os.path.join(checkpoint_path, "CNN_SVM"), global_step=index)
    #print("sdfsdfsdfsdf")  
    test_features = x_test
    test_labels =  y_test
    test_labels[test_labels == 0] = -1

    feed_dict = {x_input: test_features, y_input: test_labels, keep_prob: 1.0}

    test_accuracy = sess.run(accuracy, feed_dict=feed_dict)

    print('Test Accuracy: {}'.format(test_accuracy))<jupyter_output>INFO:tensorflow:Restoring parameters from /content/drive/My Drive/Colab Notebooks/model/CNN_SVM-0
step: 0, training accuracy : 0.07000000029802322, training loss : 2.774686813354492
step: 100, training accuracy : 0.14000000059604645, training loss : 0.2390929013490677
step: 200, training accuracy : 0.09000000357627869, training loss : 0.18171891570091248
step: 300, training accuracy : 0.12999999523162842, training loss : 0.17256233096122742
step: 400, training accuracy : 0.10999999940395355, training loss : 0.17761753499507904
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:966: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file APIs to delete files with this prefix.
step: 500, training accuracy : 0.14000000059604645, training loss : 0.15811564028263092
step: 600, training accuracy : 0.15000000596046448, training l[...]
{'fraction_non_alphanumeric': 0.11139329188195113, 'fraction_numerical': 0.04221035116910413, 'mean_word_length': 4.77350859453994, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 4, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '42025', 'license': 'Apache-2.0', 'n_tokens_mistral': 4281, 'n_tokens_neox': 3895, 'n_words': 914, 'path': 'Assignmen_32_32_3.ipynb', 'repo_name': 'NZtechpapa/CIFAR10-Classifier', 'size': '11426'}