<jupyter_start><jupyter_text><jupyter_code>import torch
import torchvision
from torchvision import transforms
from PIL import Image
from torch.utils.data import Dataset
from torch.utils.data import DataLoader
import os
import shutil
Npath='/content/drive/MyDrive/Negative'
Ppath='/content/drive/MyDrive/Positive'
Nfiles=[f for f in os.listdir(Npath)]
Pfiles=[f for f in os.listdir(Ppath)]
#train_data
class trainDataset(Dataset):
    def __init__(self,Nfiles,Pfiles):
      #创建一个list用来储存图片和标签信息
      imgs = []
      for i in range(60):
        imgs.append((Npath+'/'+Nfiles[i],0))
      for file in Pfiles:
        imgs.append((Ppath+'/'+file,1))
      self.imgs = imgs
	#返回数据集大小
    def __len__(self):
        return len(self.imgs)
	#打开index对应图片进行预处理后return回处理后的图片和标签
    def __getitem__(self, index):
        pic,label = self.imgs[index]
        pic = Image.open(pic).convert('RGB')
        pic = transforms.Resize([256,256])(pic)
        pic = transforms.ToTensor()(pic)
        return pic,label
#实例化对象
data = trainDataset(Nfiles,Pfiles)
#将数据集导入DataLoader，进行shuffle以及选取batch_size
data_loader = DataLoader(data,batch_size=100,shuffle=True,num_workers=0)
import numpy as np
a=np.arange(10)
np.random.shuffle(a)
print(a)
import numpy as np
#test_data
class testDataset(Dataset):
    def __init__(self,Nfiles,Pfiles):
      imgs = []
      index=np.arange(50)
      np.random.shuffle(index)
      for i in index[0:20]: #取10个0到50的随机数
        imgs.append((Npath+'/'+Nfiles[i+40],0))
      np.random.shuffle(index)
      for i in index[0:20]:
        imgs.append((Ppath+'/'+Pfiles[i],1))
      self.imgs = imgs
    def __len__(self):
        return len(self.imgs)
    def __getitem__(self, index):
        pic,label = self.imgs[index]
        pic = Image.open(pic).convert('RGB')
        pic = transforms.Resize([256,256])(pic)
        pic = transforms.ToTensor()(pic)
        return pic,label
test_data = testDataset(Nfiles,Pfiles)
test_data_loader = DataLoader(test_data,batch_size=50,shuffle=True,num_workers=0)
for pics,label in data_loader:
    continue
for pics,label in test_data_loader:
    continue
import torch 
import torch.nn as nn
from torch.autograd import Variable
import torch.utils.data as data
import matplotlib.pyplot as plt
class CNN(nn.Module):
    def __init__(self):
        super(CNN, self).__init__()
        #卷积层1
        self.conv1 = nn.Sequential(     #input shape (3,256,256)
            nn.Conv2d(in_channels=3,    #input height
                 out_channels=16,   #n_filter
                 kernel_size=3,    #filter size
                 stride=1,       #filter step
                 padding=1       #con2d出来的图片大小不变
                 ),           #output shape (16,128,128)
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2)   #2x2采样，output shape (16,128,128)

        )
        #卷积层2
        self.conv2 = nn.Sequential(nn.Conv2d(16, 32, 5, 1, 2), #output shape (32,64,64)
                       nn.ReLU(),
                       nn.MaxPool2d(2))
        #全连接层
        self.out = nn.Linear(32*64*64,2)

    def forward(self, x):
        x = self.conv1(x)
        x = self.conv2(x)
        x = x.view(x.size(0), -1) #flat (batch_size, 32*64*64)
        output = self.out(x)
        return output
!pip install git+https://github.com/d2l-ai/d2l-zh@release
from d2l import torch as d2l
from IPython import display
% matplotlib inline
class Animator:
  def __init__(self, xlabel=None, ylabel=None, legend=None, xlim=None,
            ylim=None, xscale='linear', yscale='linear',
            fmts=('-', 'm--', 'g-.', 'r:'), nrows=1, ncols=1,
            figsize=(3.5, 2.5)):
    if legend is None:
      legend=[]
    d2l.use_svg_display()
    self.fig, self.axes = d2l.plt.subplots(nrows, ncols, figsize=figsize)
    if nrows * ncols == 1:
      self.axes = [self.axes, ]
    self.config_axes = lambda: d2l.set_axes(
      self.axes[0], xlabel, ylabel, xlim, ylim, xscale, yscale, legend)
    self.X, self.Y, self.fmts = None, None, fmts
  
  def add(self, x, y):
    # 向图表中添加多个数据点
    if not hasattr(y, "__len__"):
      y = [y]
    n = len(y)
    if not hasattr(x, "__len__"):
      x = [x] * n
    if not self.X:
      self.X = [[] for _ in range(n)]
    if not self.Y:
      self.Y = [[] for _ in range(n)]
    for i, (a, b) in enumerate(zip(x, y)):
      if a is not None and b is not None:
        self.X[i].append(a)
        self.Y[i].append(b)
    self.axes[0].cla()
    for x, y, fmt in zip(self.X, self.Y, self.fmts):
      self.axes[0].plot(x, y, fmt)
    self.config_axes()
    display.display(self.fig)
    display.clear_output(wait=True)
def get_accuracy(y_pred,y_target):
  n_correct=torch.eq(y_pred,y_target).sum().item()
  accuracy=n_correct/len(y_pred)
  return accuracy
#args
lr=0.00005
epoch_size=100
cnn = CNN()
print(cnn)
#animator
animator=Animator(xlabel='epoch', xlim=[1, epoch_size], ylim=[0, 1],
            legend=['train loss','train_acc','test_acc'])
#optimizer
optimizer = torch.optim.Adam(cnn.parameters(), lr=lr)

#loss_fun
loss_func = nn.CrossEntropyLoss()

lossbuffer=[]
trainbuffer=[]
testbuffer=[]
#training loop
for epoch in range(1):
  losses=[]
  train_accs=[]
  test_accs=[]
  for i, (x, y) in enumerate(data_loader):
    batch_x = Variable(x)
    batch_y = Variable(y)
    #输入训练数据
    output = cnn(batch_x)
    _,pred=output.max(dim=1)
    #计算误差
    print(output,batch_y)
    loss = loss_func(output, batch_y)
    #计算准确度
    acc=get_accuracy(pred,batch_y)
    losses.append(loss)
    train_accs.append(acc)
    #清空上一次梯度
    optimizer.zero_grad()
    #误差反向传递
    loss.backward()
    #优化器参数更新
    optimizer.step()
  for i, (x, y) in enumerate(test_data_loader):
    batch_x = Variable(x)
    batch_y = Variable(y)
    output = cnn(batch_x)
    _,pred=output.max(dim=1)
    acc=get_accuracy(pred,batch_y)
    test_accs.append(acc)
  lossbuffer.append(sum(losses)/len(losses))
  trainbuffer.append(sum(train_accs)/len(train_accs))
  testbuffer.append(sum(test_accs)/len(test_accs))
  if epoch%2==0:
    animator.add(epoch+1,(sum(lossbuffer)/len(lossbuffer),sum(trainbuffer)/len(trainbuffer),sum(testbuffer)/len(testbuffer)))
    lossbuffer=[]
    trainbuffer=[]
    testbuffer=[]
#lossPlot
torch.save(cnn,'/content/drive/MyDrive/Models/cnn.pt')
cnn_from_file = CNN()
print(cnn_from_file)
cnn_from_file=torch.load('/content/drive/MyDrive/Models/cnn.pt')
cnn_from_file.eval()
for i, (x, y) in enumerate(test_data_loader):
    batch_x = Variable(x)
    batch_y = Variable(y)
    output = cnn_from_file(batch_x)
    _,pred=output.max(dim=1)
    acc=get_accuracy(pred,batch_y)
    print(acc)
    #print(output)
    print(pred)
processDataPath='/content/drive/MyDrive/Unverified'
processfiles=[f for f in os.listdir(processDataPath)]
class processDataset(Dataset):
    def __init__(self,processfiles):
      img = []
      for file in processfiles:
        img.append([processDataPath+'/'+file,2,file])
      self.imgs=img
    def __len__(self):
      return len(self.imgs)
    def __getitem__(self, index):
      pic,label,name = self.imgs[index]
      pic = Image.open(pic).convert('RGB')
      pic = transforms.Resize([256,256])(pic)
      pic = transforms.ToTensor()(pic)
      return pic,label
process_data = processDataset(processfiles)
process_data_loader = DataLoader(process_data,batch_size=1,shuffle=False,num_workers=0)
for pics,label in process_data_loader:
    continue
res=[]
outputs=[]
for i, (x, y) in enumerate(process_data_loader):
    batch_x = Variable(x)
    batch_y = Variable(y)
    output = cnn_from_file(batch_x)
    _,pred=output.max(dim=1)
    res.append(pred.tolist()[0])
    outputs.append(output.tolist()[0])
outputs=np.array(outputs)
outputs
def softmax(X):
  res=[]
  for i in X:
    sum=np.exp(i[0])+np.exp(i[1])
    res.append((np.exp(i[0])/sum,np.exp(i[1])/sum))
  return res
outputs=softmax(outputs)
outputs=torch.Tensor(outputs)
possibility,pred=outputs.max(dim=1)
import pandas as pd
result=pd.DataFrame()
result['Label']=pred.tolist()
result['Possibility']=possibility.tolist()
list_t = [process_data.imgs[i][2] for i in range(len(process_data.imgs))]
result['fileName']=list_t
result.to_excel('Predict.xlsx')
result<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1430323846908734, 'fraction_numerical': 0.020608439646712464, 'mean_word_length': 3.3552350427350426, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '117874', 'license': 'MIT', 'n_tokens_mistral': 3205, 'n_tokens_neox': 3155, 'n_words': 571, 'path': 'CNN.ipynb', 'repo_name': 'marongkang/MLeveryday', 'size': '8152'}