<jupyter_start><jupyter_text>首先我们定义一个基于ConvNet的简单神经网络  1、加载数据<jupyter_code>batch_size = test_batch_size = 32  
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
mnist_data = datasets.MNIST("./mnist_data", train=True, download=True,
                           transform=transforms.Compose([
                               transforms.ToTensor(),
                           ]))
mnist_data
import numpy as np
data = [d[0].data.cpu().numpy() for d in mnist_data]
print(np.mean(data))
print(np.std(data))
#torch.utils.data.DataLoader在训练模型时使用到此函数，用来把训练数据分成多个batch(iterator)
#transform 接受一个图像返回变换后的图像的函数，相当于图像预处理
#常用的操作如 ToTensor, RandomCrop，Normalize等. 
#他们可以通过transforms.Compose被组合在一起 
#.ToTensor()将shape为(H, W, C)的nump.ndarray或img转为shape为(C, H, W)的tensor，
#其将每一个数值归一化到[0,1]，其归一化方法比较简单，直接除以255即可。
#.Normalize作用就是.ToTensor将输入归一化到(0,1)后，再使用公式”(x-mean)/std”，将每个元素分布到(-1,1)  
#shuffle随机打乱数据
#kwargs是上面gpu的设置
train_loader = torch.utils.data.DataLoader(
    datasets.MNIST('./mnist_data', 
                   train=True, #如果true，从training.pt创建数据集
                   transform=transforms.Compose([  
                       transforms.ToTensor(), 
                       transforms.Normalize((0.1307,), (0.3081,)) # 所有图片像素均值和方差
                   ])), 
    batch_size=batch_size, 
    shuffle=True,
    **kwargs)
test_loader = torch.utils.data.DataLoader(
    datasets.MNIST('./mnist_data', 
                   train=False, #如果False，从test.pt创建数据集
                   transform=transforms.Compose([
                       transforms.ToTensor(),
                       transforms.Normalize((0.1307,), (0.3081,))
                   ])),
    batch_size=test_batch_size, 
    shuffle=True, 
    **kwargs)
len(train_loader)
train_loader.dataset[0][0].shape<jupyter_output><empty_output><jupyter_text>2、定义CNN模型<jupyter_code>class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        
        #torch.nn.Conv2d(in_channels, out_channels, kernel_size, stride=1)
        #in_channels：输入图像通道数，手写数字图像为1，一般彩色图像为3
        #out_channels：输出通道数，等于卷积核的数量
        #kernel_size：卷积核大小
        #stride：步长
        self.conv1 = nn.Conv2d(1, 20, 5, 1)

        #上个卷积网络的out_channels，就是下一个网络的in_channels，所以这里是20
        #out_channels：卷积核数量50    
        self.conv2 = nn.Conv2d(20, 50, 5, 1)

        #全连接层torch.nn.Linear(in_features, out_features)
        #in_features:输入特征维度，4*4*50是自己算出来的，跟输入图像维度有关
        #out_features；输出特征维度      
        self.fc1 = nn.Linear(4*4*50, 500)

        #输出维度10，10分类
        self.fc2 = nn.Linear(500, 10)

    def forward(self, x):  
        x = F.relu(self.conv1(x)) # x = (32,50,24,24)
        x = F.max_pool2d(x, 2, 2) # x = (32,50,12,12)
        x = F.relu(self.conv2(x)) # x = (32,50,8,8)
        x = F.max_pool2d(x, 2, 2) # x = (32,50,4,4)
        x = x.view(-1, 4*4*50)    # x = (32,4*4*50)
        x = F.relu(self.fc1(x))   # x = (32,4*4*50)*(4*4*50, 500)=(32,500)
        x = self.fc2(x)           # x = (32,500)*(500, 10)=(32,10)
        return F.log_softmax(x, dim=1)  #log probability
model = Net()
batch = next(iter(train_loader))
data = batch[0]
batch[1].shape
batch[0].shape
data = F.relu(model.conv1(data))
data = F.max_pool2d(data,2,2)
data = F.relu(model.conv2(data))
data = F.max_pool2d(data,2,2)
data = data.view(-1,4*4*50)
data = F.relu(model.fc1(data))
data = model.fc2(data)
data = F.log_softmax(data, dim=1)
data.shape
data.argmax(dim=1, keepdim=True)
pred = data.argmax(dim=1, keepdim=True)
target = batch[1]
correct = pred.eq(target.view_as(pred)).sum().item()
correct<jupyter_output><empty_output><jupyter_text>3、初始化模型和定义优化函数<jupyter_code>lr = 0.01
momentum = 0.5
model = Net().to(device) #模型初始化
optimizer = optim.SGD(model.parameters(), lr=lr, momentum=momentum) #定义优化器<jupyter_output><empty_output><jupyter_text>4、定义训练和测试模型 NLL loss的定义$\ell(x, y) = L = \{l_1,\dots,l_N\}^\top, \quad        l_n = - w_{y_n} x_{n,y_n}, \quad        w_{c} = \text{weight}[c] \cdot \mathbb{1}\{c \not= \text{ignore_index}\}$<jupyter_code>def train(model, device, train_loader, optimizer, epoch, log_interval=10000):
    model.train() #进入训练模式
    for batch_idx, (data, target) in enumerate(train_loader):
        #data:[32,1,28,28] target:[32]
        data, target = data.to(device), target.to(device)
        
        output = model(data)  #[32,10]
        #F.nll_loss(F.log_softmax(input), target) ：
        #单分类交叉熵损失函数，一张图片里只能有一个类别，输入input的需要softmax
        #还有一种是多分类损失函数，一张图片有多个类别，输入的input需要sigmoid
        loss = F.nll_loss(output, target) #这里loss求的是平均数，除以了batch

        optimizer.zero_grad() #梯度归零
        loss.backward()
        optimizer.step()
        
        if batch_idx % log_interval == 0:
            print("Train Epoch: {} [{}/{} ({:0f}%)]\tLoss: {:.6f}".format(
                epoch, 
                batch_idx * len(data),
                len(train_loader.dataset), #60000
                100. * batch_idx / len(train_loader), #len(train_loader)=60000/32=1875
                loss.item()
            ))
def test(model, device, test_loader):
    model.eval() #进入测试模式
    test_loss = 0
    correct = 0
    with torch.no_grad():
        for data, target in test_loader:
            data, target = data.to(device), target.to(device)
            output = model(data) 
            test_loss += F.nll_loss(output, target, reduction='sum').item()
            #reduction='sum'代表batch内的每个元素loss累加求和，默认是mean求平均
                       
            pred = output.argmax(dim=1, keepdim=True)
            correct += pred.eq(target.view_as(pred)).sum().item()

    test_loss /= len(test_loader.dataset)

    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        100. * correct / len(test_loader.dataset)))<jupyter_output><empty_output><jupyter_text>5、查看运行结果<jupyter_code>epochs = 2
for epoch in range(1, epochs + 1):
    train(model, device, train_loader, optimizer, epoch)
    test(model, device, test_loader)

save_model = True
if save_model:
    torch.save(model.state_dict(),"mnist_cnn.pt") 
    #词典格式，model.state_dict()只保存模型参数<jupyter_output>Train Epoch: 1 [0/60000 (0.000000%)]	Loss: 0.056703

Test set: Average loss: 0.0448, Accuracy: 9865/10000 (99%)

Train Epoch: 2 [0/60000 (0.000000%)]	Loss: 0.007774

Test set: Average loss: 0.0303, Accuracy: 9904/10000 (99%)<jupyter_text>CNN模型的迁移学习 - 很多时候当我们需要训练一个新的图像分类任务，我们不会完全从一个随机的模型开始训练，而是利用预训练的模型来加速训练的过程。我们经常使用在`ImageNet`上的预训练模型。- 这是一种transfer learning的方法。我们常用以下两种方法做迁移学习。    - fine tuning: 从一个预训练模型开始，我们改变一些模型的架构，然后继续训练整个模型的参数。    - feature extraction: 我们不再改变与训练模型的参数，而是只更新我们修改的部分模型(fc)参数。我们之所以叫它feature extraction是因为我们把预训练的CNN模型当做一个特征提取模型，利用提取出来的特征做来完成我们的训练任务。    以下是构建和训练迁移学习模型的基本步骤：- 初始化预训练模型- 把最后一层的输出层改变成我们想要分的类别总数- 定义一个optimizer来更新参数- 模型训练<jupyter_code>import numpy as np
import torchvision
from torchvision import datasets, transforms, models

import matplotlib.pyplot as plt
import time
import os
import copy
print("Torchvision Version: ",torchvision.__version__)<jupyter_output>Torchvision Version:  0.5.0+cu100<jupyter_text>数据------我们会使用*hymenoptera_data*数据集，[下载](https://download.pytorch.org/tutorial/hymenoptera_data.zip).这个数据集包括两类图片, **bees** 和 **ants**, 这些数据都被处理成了可以使用`ImageFolder `来读取的格式。我们只需要把``data_dir``设置成数据的根目录，然后把``model_name``设置成我们想要使用的与训练模型：::   [resnet, alexnet, vgg, squeezenet, densenet, inception]其他的参数有：- ``num_classes``表示数据集分类的类别数- ``batch_size``- ``num_epochs``- ``feature_extract``表示我们训练的时候使用fine tuning还是feature extraction方法。如果``feature_extract = False``，整个模型都会被同时更新。如果``feature_extract = True``，只有模型的最后一层被更新。 1、查看数据，只是查看作用---------<jupyter_code># Top level data directory. Here we assume the format of the directory conforms to the ImageFolder structure
data_dir = "./hymenoptera_data"
# Batch size for training (change depending on how much memory you have)
batch_size = 32
# 输入图像维度
input_size = 224

#os.path.join() 连接路径，相当于.../data_dir/train
all_imgs = datasets.ImageFolder(os.path.join(data_dir, "train"),
                                transforms.Compose([
        transforms.RandomResizedCrop(input_size), #把每张图片截成resnet需要输入的维度224
        transforms.RandomHorizontalFlip(),#水平翻转
        transforms.ToTensor(),
    ]))
loader = torch.utils.data.DataLoader(all_imgs, batch_size=batch_size, shuffle=True, num_workers=4)
len(next(iter(loader)))
img = next(iter(loader))[0] #这个img是一个batch的tensor
img.shape
unloader = transforms.ToPILImage()  # reconvert into PIL image

def imshow(tensor, title=None):
    image = tensor.cpu().clone()  # we clone the tensor to not do changes on it
    image = unloader(image) #tensor转换成图像
    plt.imshow(image)
    if title is not None:
        plt.title(title)
    plt.show()
        
plt.figure()
imshow(img[8], title='Image')<jupyter_output><empty_output><jupyter_text>3、把训练集和验证集分batch转换成迭代器---------现在我们知道了模型输入的size，我们就可以把数据预处理成相应的格式。<jupyter_code>data_transforms = {
    "train": transforms.Compose([
        transforms.RandomResizedCrop(input_size),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
    "val": transforms.Compose([
        transforms.Resize(input_size),
        transforms.CenterCrop(input_size),#中间截取
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
}

#train,val datasets dataloaders合为一个dict
# Create training and validation datasets
image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x]) 
                  for x in ['train', 'val']}
# Create training and validation dataloaders
dataloaders_dict = {x: torch.utils.data.DataLoader(image_datasets[x], 
                                    batch_size=batch_size, shuffle=True, num_workers=4) for x in ['train', 'val']}
#把迭代器存放到字典里作为value，key是train和val，后面调用key即可。

print("Datasets and Dataloaders Initialized")
inputs, labels=next(iter(dataloaders_dict["train"]))
print(inputs.shape)
print(labels)
#其实最后一个batch不足32<jupyter_output>torch.Size([32, 3, 224, 224])
tensor([0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0,
        1, 1, 1, 1, 0, 0, 1, 0])<jupyter_text>4、加载resnet模型并修改全连接层<jupyter_code>num_classes = 2
num_epochs = 2
# Flag for feature extracting. 
# When False, we finetune the whole model, when True we only update the reshaped layer params
feature_extract = True  #只更新修改的层
def set_parameter_requires_grad(model, feature_extracting):
    if feature_extracting:
        for param in model.parameters():
            param.requires_grad = False
def initialize_model(num_classes, feature_extract, use_pretrained=True):#如果True，返回预训练好的模型和参数
    model_ft = models.resnet18(pretrained=use_pretrained) 
    set_parameter_requires_grad(model_ft, feature_extract)

    num_ftrs = model_ft.fc.in_features 
        #model_ft.fc是resnet的最后全连接层
        #(fc): Linear(in_features=512, out_features=1000, bias=True)
        #in_features 是全连接层的输入特征维度
    model_ft.fc = nn.Linear(num_ftrs, num_classes)
    input_size = 224 #resnet网络输入图片维度是224
    return model_ft, input_size 
    
model_ft, input_size = initialize_model(num_classes, feature_extract, use_pretrained=True)
print(model_ft)
model_ft.fc.weight.requires_grad
model_ft.layer1[0].conv1.weight.requires_grad
models.resnet18().fc
model_ft.fc<jupyter_output><empty_output><jupyter_text>5、查看需要更新的参数、定义优化器<jupyter_code>next(iter(model_ft.named_parameters()))
len(next(iter(model_ft.named_parameters()))) #是元组，只有两个值
for name,param in model_ft.named_parameters():
    print(name) #看下都有哪些参数
model_ft = model_ft.to(device)

#  Gather the parameters to be optimized/updated in this run. If we are finetuning 
#  we will be updating all parameters. However, if we are doing 
#  feature extract method, we will only update the parameters
#  that we have just initialized, i.e. the parameters with requires_grad is True.

params_to_update = model_ft.parameters() #需要更新的参数
print("Params to learn:")
if feature_extract:
    params_to_update = [] #需要更新的参数存放在此
    for name,param in model_ft.named_parameters(): 
        if param.requires_grad == True: 
            params_to_update.append(param)
            print("\t",name)
else:
    for name,param in model_ft.named_parameters():
        if param.requires_grad == True:
            print("\t",name)
optimizer_ft = optim.SGD(params_to_update, lr=0.001, momentum=0.9)
criterion = nn.CrossEntropyLoss()
data = next(iter(dataloaders_dict['train']))
inputs = data[0]
lables = data[1]
inputs = inputs.cuda(1)
model_ft(inputs).shape<jupyter_output><empty_output><jupyter_text>6、定义训练函数<jupyter_code>def train_eval_model(model, dataloaders, criterion, optimizer, num_epochs=5):
    start = time.time()
    val_acc_history = [] 
    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.
    
    for epoch in range(num_epochs):
        print("Epoch {}/{}".format(epoch, num_epochs-1))
        print("-"*10)
        
        for phase in ["train", "val"]:
            running_loss = 0.
            running_corrects = 0.
            if phase == "train":
                model.train()
            elif phase == 'val': 
                model.eval()
            
            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device)
                labels = labels.to(device)
                
                with torch.autograd.set_grad_enabled(phase=="train"):
                    #torch.autograd.set_grad_enabled梯度管理器，可设置为打开或关闭
                    #phase=="train"返回值True和False，双等号要注意
                    outputs = model(inputs)
                    loss = criterion(outputs, labels)
                    
                _, preds = torch.max(outputs, 1)
                #返回每一行最大的数和索引，preds的位置是索引的位置
                #也可以preds = outputs.argmax(dim=1)
                
                if phase == "train":
                    optimizer.zero_grad()
                    loss.backward()
                    optimizer.step()
                    
                running_loss += loss.item() * inputs.size(0) #交叉熵损失函数是平均过的
                running_corrects += torch.sum(preds.view(-1) == labels.view(-1)).item()
            
            epoch_loss = running_loss / len(dataloaders[phase].dataset)
            epoch_acc = running_corrects / len(dataloaders[phase].dataset)
       
            print("{} Loss: {} Acc: {}".format(phase, epoch_loss, epoch_acc))
            if phase == "val" and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())
                #模型变好，就拷贝更新后的模型参数
                
            if phase == "val":
                val_acc_history.append(epoch_acc) #记录每个epoch验证集的准确率
                
    time_elapsed = time.time() - start
    print("Training compete in {}m {}s".format(time_elapsed // 60, time_elapsed % 60))
    print("Best val Acc: {}".format(best_acc))
    
    model.load_state_dict(best_model_wts) #把最新的参数复制到model中
    return model, val_acc_history<jupyter_output><empty_output><jupyter_text>7、运行模型<jupyter_code># Train and evaluate
model_ft, ohist = train_eval_model(model_ft, dataloaders_dict, criterion, optimizer_ft, num_epochs=num_epochs)
ohist
model_ft
# Initialize the non-pretrained version of the model used for this run
scratch_model,_ = initialize_model(model_name, 
                                   num_classes, 
                                   feature_extract=False, #所有参数都训练
                                   use_pretrained=False)# 不要imagenet的参数
scratch_model = scratch_model.to(device)
scratch_optimizer = optim.SGD(scratch_model.parameters(), 
                              lr=0.001, momentum=0.9)
scratch_criterion = nn.CrossEntropyLoss()
_,scratch_hist = train_eval_model(scratch_model, 
                             dataloaders_dict, 
                             scratch_criterion, 
                             scratch_optimizer, 
                             num_epochs=num_epochs)

# Plot the training curves of validation accuracy vs. number 
#  of training epochs for the transfer learning method and
#  the model trained from scratch
# ohist = []
# shist = []

# ohist = [h.cpu().numpy() for h in ohist]
# shist = [h.cpu().numpy() for h in scratch_hist]

plt.title("Validation Accuracy vs. Number of Training Epochs")
plt.xlabel("Training Epochs")
plt.ylabel("Validation Accuracy")
plt.plot(range(1,num_epochs+1),ohist,label="Pretrained")
plt.plot(range(1,num_epochs+1),scratch_hist,label="Scratch")
plt.ylim((0,1.))
plt.xticks(np.arange(1, num_epochs+1, 1.0))
plt.legend()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12815545735213996, 'fraction_numerical': 0.030207639687632425, 'mean_word_length': 3.134134134134134, 'pattern_counts': {'":': 5, '<': 46, '<?xml version=': 0, '>': 47, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '496084', 'license': 'MIT', 'n_tokens_mistral': 6806, 'n_tokens_neox': 6650, 'n_words': 1013, 'path': '5/CNN-Image-Classification-comments.ipynb', 'repo_name': 'TaoWen0309/pytorch-learning', 'size': '16519'}