<jupyter_start><jupyter_text>数据预处理<jupyter_code>import os
import cv2
import h5py
import numpy as np
import pandas as pd
from keras.applications import Xception, xception
from keras.models import Model, Sequential
from keras.layers import Dropout, Dense, Input, Lambda, Flatten, Conv2D, MaxPooling2D
from sklearn.model_selection import train_test_split
from keras.optimizers import SGD
from tqdm import tqdm
data_path_train = '../dogs-vs-cats-dataset/train'
data_path_train_extra = '../dogs-vs-cats-dataset/images-Oxford-IIIT'
data_path_test = '../dogs-vs-cats-dataset/test'
image_names_train = os.listdir(data_path_train)
image_names_train_extra = os.listdir(data_path_train_extra)
image_names_test = os.listdir(data_path_test)
input_shape = (224, 224, 3)
labels = []
trains = []
tests = []


# 处理标准的训练数据
for i in tqdm(range(len(image_names_train))):
    image_name = image_names_train[i]
    image_path = os.path.join(data_path_train, image_name)
    image = cv2.imread(image_path)
    if image is None:
        print('Read train image failed:', image_path)
        continue
    image = cv2.resize(image, (input_shape[0], input_shape[1]))
    image = image / 255
    trains.append(image[:, :, ::-1])
    # cat: 0, dog: 1
    category = 1 if 'dog' in image_name else 0
    labels.append(category)

    
# # 猫的种类
# cat_types = ['Abyssinian', 'Bengal', 'Birman', 'Bombay', 'British_Shorthair', 'Egyptian_Mau', 'Maine_Coon', 'Persian',
#              'Ragdoll', 'Russian_Blue', 'Siamese', 'Sphynx']
# # 处理扩展的训练数据
# for i in tqdm(range(len(image_names_train_extra))):
#     image_name = image_names_train_extra[i]
#     image_path = os.path.join(data_path_train_extra, image_name)
#     image = cv2.imread(image_path)
#     if image is None:
#         print('Read extra train image failed:', image_path)
#         continue
#     image = cv2.resize(image, (input_shape[0], input_shape[1]))
#     image = image / 255
#     index = len(image_names_train) + i
#     trains.append(image[:, :, ::-1])
    
#     # 获取动物的种类（dog or cat）
#     spt = image_names_train_extra[i].split('_')
#     spt.pop()
#     tp = '_'.join(spt)
#     category = 0 if tp in cat_types else 1
#     labels.append(category)
    

# 处理标准的测试数据
for i in tqdm(range(len(image_names_test))):
    image_name = image_names_test[i]
    image_path = os.path.join(data_path_test, image_name)
    image = cv2.imread(image_path)
    if image is None:
        print('Read test image failed:', image_path)
        continue
    image = cv2.resize(image, (input_shape[0], input_shape[1]))
    image = image / 255
    tests.append(image[:, :, ::-1])
    
    
trains = np.array(trains, dtype='float16')
labels = np.array(labels, dtype='float16')
tests = np.array(tests)

print('Training data size: %d' % len(trains))
print('Label size: %d' % len(labels))
print('Testing data size: %d' % len(tests))<jupyter_output>100%|██████████| 25000/25000 [01:01<00:00, 409.82it/s]
100%|██████████| 12500/12500 [00:35<00:00, 349.83it/s]<jupyter_text>构建模型基于VGG，部分层进行了调整。<jupyter_code>model = Sequential()
model.add(Conv2D(64, (3, 3), activation='relu', input_shape=input_shape))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Conv2D(256, (3, 3), activation='relu'))
model.add(Conv2D(256, (3, 3), activation='relu'))
model.add(Conv2D(256, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Conv2D(512, (3, 3), activation='relu'))
model.add(Conv2D(512, (3, 3), activation='relu'))
model.add(Conv2D(512, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Conv2D(512, (3, 3), activation='relu'))
model.add(Conv2D(512, (3, 3), activation='relu'))
model.add(Conv2D(512, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Flatten())
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))

sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])

print('Model ready!')<jupyter_output>Model ready!<jupyter_text>训练<jupyter_code>X_train, X_val, y_train, y_val = train_test_split(trains, labels, shuffle=True, test_size=0.2, random_state=2018)

model.fit(x=X_train, y=y_train, batch_size=32, epochs=10, validation_data=(X_val, y_val))<jupyter_output>Train on 20000 samples, validate on 5000 samples
Epoch 1/10
  512/20000 [..............................] - ETA: 5:43:11 - loss: 0.6932 - acc: 0.5195<jupyter_text>测试<jupyter_code>y_pred = model.predict(tests, verbose=1)
y_pred = y_pred.clip(min=0.005, max=0.995)

df = pd.read_csv("sample_submission.csv")

for i in range(len(image_names_test)):
    image_name = image_names_test[i]
    index = int(str.split(image_name, '.')[0]) - 1
    df.iat[index, 1] = y_pred[i]

df.to_csv('predict_vgg.csv', index=None)
print('The prediction result has been wrote to predict_vgg.csv')<jupyter_output>15/15 [==============================] - 6s 372ms/step
The prediction result has been wrote to predict.csv
{'fraction_non_alphanumeric': 0.17569601203912716, 'fraction_numerical': 0.05304740406320542, 'mean_word_length': 5.0420454545454545, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '401141', 'license': 'MIT', 'n_tokens_mistral': 2279, 'n_tokens_neox': 2130, 'n_words': 391, 'path': 'with_VGG_ConvNet.ipynb', 'repo_name': 'dongrenguang/dogs-vs-cats', 'size': '5316'}