<jupyter_start><jupyter_text>**解压数据集**<jupyter_code>!unzip -qo data/data89598/garbage.zip -d data<jupyter_output><empty_output><jupyter_text>**将json文件读出为字典，方便识别垃圾分类**<jupyter_code>f = open("data/garbage/garbage_classify_rule.json","r")
file_dict = json.load(f)
print(file_dict)
f.close()<jupyter_output>{'0': '其他垃圾/一次性快餐盒', '1': '其他垃圾/污损塑料', '2': '其他垃圾/烟蒂', '3': '其他垃圾/牙签', '4': '其他垃圾/破碎花盆及碟碗', '5': '其他垃圾/竹筷', '6': '厨余垃圾/剩饭剩菜', '7': '厨余垃圾/大骨头', '8': '厨余垃圾/水果果皮', '9': '厨余垃圾/水果果肉', '10': '厨余垃圾/茶叶渣', '11': '厨余垃圾/菜叶菜根', '12': '厨余垃圾/蛋壳', '13': '厨余垃圾/鱼骨', '14': '可回收物/充电宝', '15': '可回收物/包', '16': '可回收物/化妆品瓶', '17': '可回收物/塑料玩具', '18': '可回收物/塑料碗盆', '19': '可回收物/塑料衣架', '20': '可回收物/快递纸袋', '21': '可回收物/插头电线', '22': '可回收物/旧衣服', '23': '可回收物/易拉罐', '24': '可回收物/枕头', '25': '可回收物/毛绒玩具', '26': '可回收物/洗发水瓶', '27': '可回收物/玻璃杯', '28': '可回收物/皮鞋', '29': '可回收物/砧板', '30': '可回收物/纸板箱', '31': '可回收物/调料瓶', '32': '可回收物/酒瓶', '33': '可回收物/金属食品罐', '34': '可回收物/锅', '35': '可回收物/食用油桶', '36': '可回收物/饮料瓶', '37': '有害垃圾/干电池', '38': '有害垃圾/软膏', '39': '有害垃圾/过期药物'}<jupyter_text>**创建数据集**<jupyter_code>class GarbageDataset(Dataset):
    def __init__(self,transforms=None,train="train"):
        super().__init__()
        self.transforms=transforms
        self.labels=list()
        self.datas=list()
        file_path=os.path.join("data/garbage",train) 
        #遍历train下的文件夹
        for file_name in os.listdir(file_path):  #file_name就是label,0、1、2、3、4、5
            sub_file_path=os.path.join(file_path,file_name) 
            #遍历子文件夹下的每一张图片
            for img_name in os.listdir(sub_file_path):
                if "DeepLearning - ┐ь╜▌╖╜╩╜.lnk" in img_name or "img_12074.jpg" in img_name:
                    continue
                img_path=os.path.join(sub_file_path,img_name)  
                #读取图片
                im=np.array(Image.open(img_path))
                if im is None:  #图不为空
                    continue
                if im.ndim==3:  #图片维度为3
                    self.datas.append(im)
                    self.labels.append(file_name)  #加入label

    def __getitem__(self,index):
        data=self.datas[index]
        if self.transforms!=None:
            data=self.transforms(data)
        label=int(self.labels[index])#np.array([int(self.labels[index])])
        x=paddle.to_tensor(label)
        y_data = paddle.cast(x, 'int64')
        y_data = paddle.reshape(y_data, (-1, 1))
        return data,label

    def __len__(self):
        return len(self.labels)

transforms=T.Compose([T.Resize([224,224]),T.ToTensor()])
train_dataset=GarbageDataset(transforms=transforms) # 训练集
test_dataset=GarbageDataset(transforms=transforms,train="test") # 测试集
valid_dataset = GarbageDataset(transforms=transforms,train="valid") # 验证集<jupyter_output><empty_output><jupyter_text>**加载数据集**<jupyter_code>TrainDataLoader=DataLoader(dataset=train_dataset,shuffle=True,batch_size=100)
TestDataLoader=DataLoader(dataset=test_dataset,shuffle=True,batch_size=100)<jupyter_output><empty_output><jupyter_text>**resnet网络结构**<jupyter_code>class IndentityBlock(paddle.nn.Layer):
    def __init__(self,in_channels,filters):
        super().__init__()
        F1,F2=filters
        self.stage=paddle.nn.Sequential(
            paddle.nn.Conv2D(in_channels=in_channels,out_channels=F1,kernel_size=3,stride=1,padding="same"),
            paddle.nn.BatchNorm2D(F1),
            paddle.nn.ReLU(),
            paddle.nn.Conv2D(in_channels=F1,out_channels=F2,kernel_size=3,stride=1,padding="same"),
            paddle.nn.BatchNorm2D(F2)
        )
        self.relu=paddle.nn.ReLU()

    def forward(self,x):
        shortcut_x=x
        x=self.stage(x)
        x=x+shortcut_x
        x=self.relu(x)
        return x

ib=IndentityBlock(64,(64,64))
paddle.summary(ib,(1,64,128,128))


class ConvBlock(paddle.nn.Layer):
    def __init__(self,in_channels,filters,stride=1):
        super().__init__()
        F1,F2=filters
        self.stage=paddle.nn.Sequential(
            paddle.nn.Conv2D(in_channels=in_channels,out_channels=F1,kernel_size=3,stride=stride,padding="same"),
            paddle.nn.BatchNorm2D(F1),
            paddle.nn.ReLU(),
            paddle.nn.Conv2D(in_channels=F1,out_channels=F2,kernel_size=3,stride=1,padding="same"),
            paddle.nn.BatchNorm2D(F2)
        )
        self.shortcut_x=paddle.nn.Sequential(
            paddle.nn.Conv2D(in_channels=in_channels,out_channels=F2,kernel_size=1,stride=stride,padding="same"),
            paddle.nn.BatchNorm2D(F2)
        )
        self.relu=paddle.nn.ReLU()

    def forward(self,x):
        shortcut_x=self.shortcut_x(x)
        x=self.stage(x)
        x=x+shortcut_x
        x=self.relu(x)
        return x

cb=ConvBlock(64,(64,64),2)
paddle.summary(cb,(1,64,128,128))

class ResNet34(paddle.nn.Layer):
    def __init__(self):
        super().__init__()
    
        self.conv1 = paddle.nn.Sequential(
            paddle.nn.Conv2D(in_channels=3,out_channels=64,kernel_size=7,stride=2,padding="same"),
            paddle.nn.ReLU(),
            paddle.nn.BatchNorm2D(64),
            paddle.nn.MaxPool2D(kernel_size=3,stride=2)
        )
        self.conv2 = paddle.nn.Sequential(
            IndentityBlock(64,(64,64)),
            IndentityBlock(64,(64,64)),
            IndentityBlock(64,(64,64))
        )
        self.conv3 = paddle.nn.Sequential(
            ConvBlock(64,(128,128),stride=2),
            IndentityBlock(128,(128,128)),
            IndentityBlock(128,(128,128)),
            IndentityBlock(128,(128,128))
        )
        self.conv4 = paddle.nn.Sequential(
            ConvBlock(128,(256,256),stride=2),
            IndentityBlock(256,(256,256)),
            IndentityBlock(256,(256,256)),
            IndentityBlock(256,(256,256)),
            IndentityBlock(256,(256,256)),
            IndentityBlock(256,(256,256))
        )
        self.conv5 = paddle.nn.Sequential(
            ConvBlock(256,(512,512),stride=2),
            IndentityBlock(512,(512,512)),
            IndentityBlock(512,(512,512))
        )
        self.fc = paddle.nn.Sequential(
            paddle.nn.AvgPool2D(512),
            paddle.nn.Flatten(),
            paddle.nn.Dropout(0.5),
            paddle.nn.Linear(in_features=512*1*1,out_features=40)
        )


    def forward(self,x):
    
        x = self.conv1(x)
        x = self.conv2(x)
        x = self.conv3(x)
        x = self.conv4(x)
        x = self.conv5(x)
        x = self.fc(x)
        return x
    
resnet=ResNet34()
paddle.summary(resnet,(1,3,224,224))<jupyter_output>-----------------------------------------------------------------------------
  Layer (type)        Input Shape          Output Shape         Param #    
=============================================================================
    Conv2D-11      [[1, 3, 224, 224]]   [1, 64, 112, 112]        9,472     
     ReLU-9       [[1, 64, 112, 112]]   [1, 64, 112, 112]          0       
 BatchNorm2D-11   [[1, 64, 112, 112]]   [1, 64, 112, 112]         256      
   MaxPool2D-1    [[1, 64, 112, 112]]    [1, 64, 55, 55]           0       
    Conv2D-12      [[1, 64, 55, 55]]     [1, 64, 55, 55]        36,928     
 BatchNorm2D-12    [[1, 64, 55, 55]]     [1, 64, 55, 55]          256      
     ReLU-10       [[1, 64, 55, 55]]     [1, 64, 55, 55]           0       
    Conv2D-13      [[1, 64, 55, 55]]     [1, 64, 55, 55]        36,928     
 BatchNorm2D-13    [[1, 64, 55, 55]]     [1, 64, 55, 55]          256      
     ReLU-11       [[1, 64, 55, 55]]     [1, 64, 55, 55]           0       
Indentit[...]<jupyter_text>**训练模型并保持**<jupyter_code>model=paddle.Model(resnet)
model.prepare(optimizer=paddle.optimizer.Adam(parameters=model.parameters()),loss=paddle.nn.CrossEntropyLoss(),metrics=paddle.metric.Accuracy())
vsDL=paddle.callbacks.VisualDL("dir_log"+time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) # 打印日志，方便过后查看损失率与准确率
model.fit(train_data=TrainDataLoader,epochs=40,callbacks=vsDL,verbose=1) # 训练集训练
model.evaluate(eval_data=TestDataLoader,verbose=1) # 测试集预测
model.save("mymodel/resnet") # 保存模型
# 加载模型
model=paddle.Model(resnet)
model.prepare(optimizer=paddle.optimizer.Adam(parameters=model.parameters()),
                loss=paddle.nn.CrossEntropyLoss(),metrics=paddle.metric.Accuracy())
model.load("mymodel/resnet")
#对验证集进行预测
result=model.predict(valid_dataset)
#使用混淆矩阵查看预测结果与真实值
#生成标签矩阵
labels = []
for i in range(40):
    labels.append(i)
def F1(x):
    return np.argmax(x[0])
yp=list(map(F1,result[0]))  #利用函数式编程，整理预测值
y_pred=yp
def F2(x):
    return x[1]
yt=list(map(F2,valid_dataset)) #利用函数式编程，整理真实值
y_true=yt
from sklearn.metrics import confusion_matrix,classification_report
import pandas as pd
import seaborn as sns
#混淆矩阵的制作
C = confusion_matrix(y_true=y_true,y_pred=y_pred,labels=labels)
#混淆矩阵信息
print("混淆矩阵信息")
print(C)
#把混淆矩阵C加入index和colums，封装成dataframe数据格式
df=pd.DataFrame(C,index=labels,columns=labels)
#展示热力图
sns.heatmap(C,annot=True)
print("计算各项指标")
#计算各项指标
print(classification_report(y_true=y_true,y_pred=y_pred,digits=4))<jupyter_output>混淆矩阵信息
[[ 4  0  0 ...  1  1  1]
 [ 0 12  1 ...  0  1  0]
 [ 0  0 19 ...  0  0  0]
 ...
 [ 0  1  1 ... 18  2  0]
 [ 0  0  0 ...  1 18  1]
 [ 0  0  1 ...  0  6 17]]<jupyter_text>**预测新图片**<jupyter_code>#载入图像方法
def image_load(img_path):
    photo=Image.open(img_path)
    photo=photo.resize([224,224])  #图像统一大小
    im=np.array(photo).astype("float32")    #图像数字化
    # im=np.where(im > 150, 0, im)        #把像素点大于150的置为0
    im=im/255                          #归一化
    im=paddle.to_tensor(im)            #张量
    return im
im=image_load("data/garbage/valid/29/img_14911.jpg")     #调用载入图像方法
plt.imshow(im)      #显示图像
im=im.reshape([1,3,224,224])   #图像变形为1张1通道28*28大小
model.load("mymodel2/resnet")   #加载模型参数
result=model.predict_batch(im)  #使用model.predict_batch对图像进行预测
print(file_dict[f"{np.argmax(result)}"])<jupyter_output>可回收物/砧板
{'fraction_non_alphanumeric': 0.1792085298339143, 'fraction_numerical': 0.06715193766659831, 'mean_word_length': 2.8496448303078137, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '463034', 'license': 'Apache-2.0', 'n_tokens_mistral': 4646, 'n_tokens_neox': 4360, 'n_words': 509, 'path': 'Garbage_classification.ipynb', 'repo_name': 'guxi-kun/Garbage_classification', 'size': '9754'}