<jupyter_start><jupyter_text>笔试《一》 使用tensorflow创建CNN对fashion-mnist数据集标签进行预测<jupyter_code>from utils import mnist_reader
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np

print(tf.__version__)

# 使用Github项目包含的函数读取数据
X_train, y_train = mnist_reader.load_mnist('data/fashion', kind='train')
X_test, y_test = mnist_reader.load_mnist('data/fashion', kind='t10k')<jupyter_output>1.4.0<jupyter_text>fashion-mnist数据集有10种类别，展示属于10个label中的一些图片<jupyter_code># 创建关于10种标签类别的dict
label_dict = {
    0: 'T-shirt/top',
    1: 'Trouser',
    2: 'Pullover',
    3: 'Dress',
    4: 'Coat',
    5: 'Sandal',
    6: 'Shirt',
    7: 'Sneaker',
    8: 'Bag',
    9: 'Ankle boot'
}
# 展示10种类别中的一件商品的图片
fig, axs = plt.subplots(2, 5)
images = []
for n in range(10):
    pic_index = np.argwhere(y_train == n)[0]
    images.append(axs[n//5,n%5])
    images[n].imshow(X_train[pic_index].reshape(28,28), cmap=plt.cm.gray)
    images[n].set_title(label_dict[n])
    images[n].axis('off')

plt.show()<jupyter_output><empty_output><jupyter_text>数据预处理对图像数据进行归一化处理，提高训练表现。提取train数据中的前5000作为validation set用于训练使用，对train、validation和test集的标签做one-hot encoding便于训练。把扁平化的单列图片数组信息重整为tensorflow接受的(n, width, height, channel)形状。<jupyter_code>from tensorflow.python.keras.utils import to_categorical

# 归一化数据
X_train = X_train.astype('float32') / 255
X_test = X_test.astype('float32') / 255

# 提取前5000为validation
X_train, X_valid = X_train[5000:], X_train[:5000]
y_train, y_valid = y_train[5000:], y_train[:5000]

# 设置tensor形状
X_train = X_train.reshape(X_train.shape[0], 28, 28, 1)
X_valid = X_valid.reshape(X_valid.shape[0], 28, 28, 1)
X_test = X_test.reshape(X_test.shape[0], 28, 28, 1)

# one-hot encoding标签
y_train = to_categorical(y_train, 10)
y_valid = to_categorical(y_valid, 10)
y_test = to_categorical(y_test, 10)

# 打印出特征和标签的数据信息形状
print('训练数据特征集大小为{}，训练数据标签集大小为{}'.format(X_train.shape, y_train.shape))
print('验证数据特征集大小为{}，验证数据标签集大小为{}'.format(X_valid.shape, y_valid.shape))
print('测试数据特征集大小为{}，测试数据标签集大小为{}'.format(X_test.shape, y_test.shape))
# 使用tensorflow的高级API keras包含的CNN网络模块做快速开发
from tensorflow.python.keras.layers import Conv2D, MaxPooling2D, Dropout, Activation
from tensorflow.python.keras.layers import Flatten, Dense, BatchNormalization
from tensorflow.python.keras.models import Sequential

# 设置序列模型
model = Sequential()

# 第一个卷积层，使用64个3x3的卷积核，same padding，最大池化层，BA层，Relu激活函数和20%的dropout
model.add(Conv2D(filters=64, kernel_size=[3, 3], input_shape=[28, 28, 1], padding='same'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.2))

# 第一个卷积层，使用128个3x3的卷积核，same padding，最大池化层，BA层，Relu激活函数和20%的dropout
model.add(Conv2D(filters=128, kernel_size=[3, 3], padding='same'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.2))

# 第一个卷积层，使用256个3x3的卷积核，same padding，最大池化层，BA层，Relu激活函数和20%的dropout
model.add(Conv2D(filters=256, kernel_size=[3, 3], padding='same'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.2))

# 添加扁平层
model.add(Flatten())

# 添加全连接层
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))

# 最后一层输出对10种标签的预测，使用softmax激活函数
model.add(Dense(10, activation='softmax'))

model.summary()
# 使用nadam梯度下降优化，交叉熵为损失函数，准确率为评估方法
model.compile(optimizer='nadam', loss='categorical_crossentropy', metrics=['accuracy'])
from tensorflow.python.keras.callbacks import ModelCheckpoint

# 设置100个epoch，100的batch size
epochs = 100
batch_size = 100

# 保留最佳模型
checkpointer = ModelCheckpoint(filepath = 'saved_models/weights.best.from_scratch.hdf5', 
                               verbose = 1, save_best_only = True)
# 训练模型
model.fit(X_train, y_train, 
          validation_data=(X_valid, y_valid), 
          epochs=epochs, 
          batch_size=batch_size, 
          callbacks=[checkpointer], 
          verbose=1)

model.load_weights('saved_models/weights.best.from_scratch.hdf5')

# fashion_predict = [np.argmax(model.predict(np.expand_dims(img, axis=0))) for img in X_test]

# test_accuracy = 100 * np.sum(np.array(fashion_predict) == np.argmax(y_test, axis=1))/len(fashion_predict)

loss, accuracy = model.evaluate(X_test, y_test, verbose=0)
print('训练的CNN模型测试集损失为{:4}，测试集准确率为{:4}%'.format(loss, accuracy*100))<jupyter_output>测试集损失为0.22500082306861877，测试集准确率为92.11%<jupyter_text>展示模型分类错误的前20个图像<jupyter_code>n_index = 0
miss_classified = []

# 记录前20个分类错误的项目
while len(miss_classified) < 20:
    predicted = np.argmax(model.predict(np.expand_dims(X_test[n_index], axis=0)))
    actual = np.argmax(y_test[n_index])
    if predicted != actual:
        miss_classified.append((n_index, predicted, actual))
    
    n_index +=1

# 展示错误分类的图片
fig, axs = plt.subplots(4, 5)
images = []
for n in range(20):
    pic_index = miss_classified[n]
    images.append(axs[n//5,n%5])
    images[n].imshow(X_test[pic_index[0]].reshape(28,28), cmap=plt.cm.gray)
    images[n].set_title('{}\n{}'.format(label_dict[pic_index[1]], 
                        label_dict[pic_index[2]]), color='r')
    images[n].axis('off')

fig.set_size_inches(14, 10, forward=True)
fig.tight_layout()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15084009816877478, 'fraction_numerical': 0.045497451387577874, 'mean_word_length': 6.237704918032787, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '61124', 'license': 'MIT', 'n_tokens_mistral': 2536, 'n_tokens_neox': 2542, 'n_words': 289, 'path': 'junpu_cnn.ipynb', 'repo_name': 'magiczjp123/fashion-mnist', 'size': '5297'}