<jupyter_start><jupyter_text>Blur Pooling: Concatenation helps the decoder to recover positional data. In this case, the boundary data of original image is critical. For the concatenating images' boundary to be more specific, we introduce Blur Pooling. It generates average blurred images from concatenation image. From average pooling, the boundary data of original image is blurred well. After that, original image is subtracted by blurred image so that the boundary data is emphasized.<jupyter_code>class BlurPooling(nn.Module):
    def __init__(self):
        super(BlurPooling, self).__init__()
        
        
    def forward(self, x):
        blur = torch.zeros(x.size(0), x.size(1), x.size(2), x.size(3))
        blur = blur.detach().numpy()
        img_np = x.cpu().detach().numpy()
        #blur = np.zeros([img_np.size(0),img_np.size(1), img_np.size(2), img_np.size(3)])
        #print('input_img')
        #print(img_np.shape)
        for i in range(x.size(0)):
            for j in range(x.size(1)):
                blur[i][j] = cv2.blur(img_np[i][j], (8,16))
        #blur = cv2.blur(img_np, (8,16))
        y_ = img_np - 1.05*blur
        y_ = torch.from_numpy(y_).float().to(device)
    
        return y_
a = torch.zeros([4,2,3])
#print(a[0][0][0])
b = np.zeros([4,2,3])
print(b[0].shape)
from torchsummary import summary

model = KMNet_vgg16bn(n_classes=34).to(device)
print(model)

#summary(model, input_size=(3,1024,2048))
class MaskToTensor(object):
    def __call__(self, img):
        return torch.from_numpy(np.array(img, dtype=np.int32)).long()
img_transform = transforms.Compose([
    transforms.RandomCrop(size=(1000,2000), padding_mode='constant'),
    transforms.Resize(size=(512,1024)),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.28689554, 0.32513303, 0.28389177], std=[0.18696375, 0.19017339, 0.18720214]),
])
target_transform = transforms.Compose([
    transforms.Resize(size=(512,1024)),
    MaskToTensor(),
])
def getMeanIoU(label, yhat, n_classes=34, ignore_class=[0,1,2,3,4,5,6,9,10,14,15,16,18,29,30]):
    if ignore_class==None:
        ignore_len = 0
    else:
        ignore_len = len(ignore_class)
    iou = np.zeros(n_classes-ignore_len)
    minus = 0
    for i in range(n_classes):
        if np.isin(i, ignore_class):
            minus+=1
            continue
#         print(i-minus)
        tfpn = torch.eq(yhat, i)

        tp = len(tfpn[(tfpn==True) & (label==i) &(yhat==i)])
        fp = len(tfpn[(tfpn==True) & (label!=i) & (yhat==i)])
        fn = len(tfpn[(tfpn==False) & (label==i) & (yhat!=i)])
        tn = len(tfpn[(tfpn==False) & (label!=i) & (yhat!=i)])
        
        if tp==0:
            iou[i-minus] = 0.0
        else:
            iou[i-minus] = tp/(tp+fp+fn)
#         print(iou[i-minus])
            
    mIoU = np.mean(iou)
#     print(iou)
    
    return mIoU
train_set = datasets.Cityscapes(root='/home/km/data/cityscapes', split='train', mode='fine', target_type='semantic', transform=img_transform, target_transform=target_transform)
val_set = datasets.Cityscapes(root='/home/km/data/cityscapes', split='val', mode='fine', target_type='semantic', transform=img_transform, target_transform=target_transform)
len(train_set)
train_set[0][0].shape
val_set[0][0].shape<jupyter_output><empty_output><jupyter_text>Using multi-GPUs<jupyter_code>print(torch.cuda.device_count())
if torch.cuda.device_count() > 1:
  print("Let's use", torch.cuda.device_count(), "GPUs!")
  # dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs
  model = nn.DataParallel(model)

model.to(device)
batch_size = 2
learning_rate = 0.01
train_loader = DataLoader(dataset=train_set, batch_size=batch_size, shuffle=True, drop_last=True)
val_loader = DataLoader(dataset=val_set, batch_size=10, shuffle=False, drop_last=False)
criterion = nn.CrossEntropyLoss()
optim = torch.optim.Adam(model.parameters(), lr=learning_rate)
max_iter = 90000
max_epoch = max_iter//(len(train_set)//batch_size)
step = 0
from torch.optim.lr_scheduler import _LRScheduler

class PolyLR(_LRScheduler):
    def __init__(self, optimizer, max_iter, power=0.9, last_epoch=-1):
        self.max_iter = max_iter
        self.power = power
        super(PolyLR, self).__init__(optimizer, last_epoch)
        
    def get_lr(self):
        return [base_lr*(1-self.last_epoch/self.max_iter)**self.power 
                for base_lr in self.base_lrs]
    

from torch.optim.lr_scheduler import StepLR
#scheduler = StepLR(optim, step_size=50, gamma=0.9)
scheduler = PolyLR(optim, max_iter = max_iter)<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>totalLoss = 0
trainLoss = []

best_model_wts = copy.deepcopy(model.state_dict())
best_miou = 0

model.train()
for epoch in range(max_epoch):
    for idx, (images, labels) in enumerate(train_loader):
        start = time.time()
        x, y = images.to(device), labels.to(device)
        yhat = model(x)
#         print(epoch,step)
        
        loss = criterion(yhat, y)
        totalLoss += loss.item()
        
        optim.zero_grad()
        loss.backward()
        optim.step()
        scheduler.step()
        
        if (step+1)%5 == 0:
            print('{}th Epoch, {}th Step, learning rate = {} - Loss: {}'.format(epoch+1, step+1, scheduler.get_lr()[0], loss.item()))
            trainLoss.append(totalLoss/5)
            totalLoss = 0
            
        if (step+1)%100 == 0:
            model.eval()
            mIoU = 0
            curr = 0
            with torch.no_grad():
                for idx, (images, labels) in enumerate(val_loader):
                    x, y = images.to(device), labels.to(device)
                    yhat = model(x)
                    
                    loss = criterion(yhat, y)

                    _, y_pred = torch.max(yhat.cpu(), dim=1)
                    mIoU += getMeanIoU(y.cpu(), y_pred)*len(y)
                    curr += len(y)
                    print(curr, end=' ')
                    if curr==300:
                        print()
                    
            mIoU /= len(val_set)
            print()
            print('*'*27, 'Test', '*'*27)
            print('time:{}, {}th Step, Loss: {}, Mean IoU = {:.3f}%'.format(timeSince(start), step+1, loss.item(), mIoU*100))
            print('*'*60)
            
            model.train()
        
        step+=1
        
!nvidia-smi<jupyter_output><empty_output><jupyter_text>model save<jupyter_code>PATH = '/root/km/models/KMNet/KMNet_vgg16bn.pth'
torch.save(model.state_dict(), PATH)<jupyter_output><empty_output><jupyter_text>validation<jupyter_code>model.eval()   #  model load 후 이어서 학습하기 위해서는 model.eval() 한번 호출이 필요
mIoU = 0
curr = 0
with torch.no_grad():
    for idx, (images, labels) in enumerate(val_loader):
        x, y = images.to(device), labels.to(device)
        yhat = model(x)

        loss = criterion(yhat, y)

        _, y_pred = torch.max(yhat.cpu(), dim=1)
        mIoU += getMeanIoU(y.cpu(), y_pred)*len(y)
        curr += len(y)
        print(curr, end=' ')
        if curr==300:
            print()

mIoU /= len(val_set)
print()
print('*'*27, 'Test', '*'*27)
print('time:{}, {}th Step, Loss: {}, Mean IoU = {:.3f}%'.format(timeSince(start), step+1, loss.item(), mIoU*100))
print('*'*60)
plt.plot(range(len(trainLoss)), trainLoss, marker='.')
plt.xlabel("steps")
plt.ylabel("train loss")
plt.show()<jupyter_output><empty_output><jupyter_text>test<jupyter_code>test_data = datasets.Cityscapes(root='/root/km/data/cityscapes', split='test', mode='fine', target_type='semantic', transform=img_transform, target_transform=target_transform)
test_loader = DataLoader(dataset=test_data, batch_size=10, shuffle=False, drop_last=False)
print(len(test_data))
KMNet_model = model
KMNet_model.load_state_dict(torch.load(PATH))
KMNet_model.eval()
print('<{}th step>'.format(step+1))

curr = 0
y_preds = torch.Tensor([]).type(torch.long)
with torch.no_grad():
    for idx, (images, labels) in enumerate(test_loader):
        x, y =images.to(device), labels.to(device)
        yhat = KMNet_model(x)
        
        loss = criterion(yhat, y)
        
        _, y_pred = torch.max(yhat.cpu(), dim=1)
        y_preds = torch.cat([y_preds, y_pred], dim=0)
        curr += len(y)
        if(curr%50)==0 or curr==len(test_data):
            print(curr, end=' ')
            
print("Prediction Ended")
test_img = datasets.Cityscapes(root='/root/km/data/cityscapes', split='test', mode='fine', target_type='semantic')
class TensorToMask(object):
    def __call__(self, tensor):
        mask = tensor.type(torch.uint8)
        
        return mask
class TestTransform(object):
    def __call__(self, tensor, size=(1024, 2048)):
        test_transform = transforms.Compose([
            TensorToMask(),
            transforms.ToPILImage(),
            transforms.Resize(size=size),
        ])
        
        return test_transform(tensor)
test_transform = transforms.Compose([
    TensorToMask(),
    transforms.ToPILImage(),
    transforms.Resize(size=(1024,2048)),
])

a, b = 15, 30
fig = plt.figure(figsize=(a,b))

for i in range(10):
    ax = fig.add_subplot(5, 2, i+1)
    ax.imshow(test_img[i][0])
fig = plt.figure(figsize=(a,b))

for i in range(10):
    ax = fig.add_subplot(5, 2, i+1)
    y_img = test_transform(y_preds[i])
    ax.imshow(y_img)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15522805507745266, 'fraction_numerical': 0.030120481927710843, 'mean_word_length': 2.9747755451047455, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '526202', 'license': 'MIT', 'n_tokens_mistral': 3520, 'n_tokens_neox': 3310, 'n_words': 611, 'path': 'implementation/KMNet_vgg16bn_more.ipynb', 'repo_name': 'klee30810/SemanticSegmentation', 'size': '9296'}