<jupyter_start><jupyter_text>what is mtcnn 이미지 상에서 얼굴의 bbox와 5개의 랜드마크를 detection하는 알고리즘- Non-max suppression을 사용 1. image pyramid을 만들기 위해 input image을 여러 번 scaled down각 scale의 image들이 CNN을 거친다.2. CNN을 거친 각각의 이미지에서 bbox에 대한 이미지 패치(image patches)을 추출한다.3. 이미지 패치를 stage2에서는 24x24, stage3에서는 48x48로 resize을 한다.4. 그리고 그놈들을 CNN에 넣는다.5. stage3에서는 bbox와 scores 이외에도 각 bbox에 대한 5개의 얼굴 랜드마크를 계산한다.  dataset Prepare<jupyter_code>import glob
import random
import os
data_path = 'data\image\car_license_plate\plate1\detect_recog_image'

image_path_list = glob.glob(data_path + '/*.jpg')

# train : test = 7:3
data_n = len(image_path_list)

train_idx_list = np.random.choice(data_n, int(data_n*0.7), replace = False)
test_idx_list = np.setdiff1d(np.arange(data_n), train_idx_list)

print(len(train_idx_list), len(test_idx_list))
for i in train_idx_list:
    img_path = image_path_list[i]
    img_name = img_path.split('\\')[-1]
    print(img_path, img_name)
    os.rename(img_path, data_path + '/train/' + img_name)
    

for j in test_idx_list:
    img_path = image_path_list[j]
    img_name = img_path.split('\\')[-1]
    print(img_path, img_name)
    os.rename(img_path, data_path + '/test/' + img_name)<jupyter_output><empty_output><jupyter_text>Dataset<jupyter_code>%load_ext autoreload

%autoreload 2
#from facenet_pytorch import MTCNN
import torch
import torchvision
import numpy as np
import pandas as pd
import glob
import cv2
import matplotlib.pyplot as plt

from torch.utils.data import Dataset
from torchvision import transforms
from PIL import Image<jupyter_output><empty_output><jupyter_text>Meta Data<jupyter_code>data_path = 'data\image\car_license_plate\plate1\detect_recog_image'

bbox_frame = pd.read_csv(data_path+'/plate1.csv')
print(bbox_frame.shape)
bbox_frame = bbox_frame[['name', 'x', 'y', 'w', 'h', 'x_size', 'y_size', 'text']]
bbox_frame.head()
bbox_frame.info()
bbox_frame.describe()<jupyter_output><empty_output><jupyter_text>dataset 생성- 이미지 사이즈 1050x753으로 통일<jupyter_code>class MyDataset(Dataset):
    def __init__(self, data_path, bbox_frame, transform=None):
        self.data_path = data_path
        self.transform = transform
        self.bbox_frame = bbox_frame
        
    def __getitem__(self, idx):
        path = self.data_path[idx]
        
        image = Image.open(path)
        image = image.resize((1050, 753)) # x=1050, y=753
        image_name = path.split('\\')[-1]
        
        df = bbox_frame[bbox_frame['name'] == image_name]
        x_size = df.iloc[0]['x_size']
        y_size = df.iloc[0]['y_size']
        bbox = np.array([df.iloc[0, 1:5]], dtype=np.float)

        bbox[0][0] = bbox[0][0]*(1050/x_size)
        bbox[0][2] = bbox[0][2]*(1050/x_size)
        bbox[0][1] = bbox[0][1]*(753/y_size)
        bbox[0][3] = bbox[0][3]*(753/y_size)

        #sample = {'image' : image, 'bbox' : bbox}     
        if self.transform:
            image = self.transform(image)
            #bbox = self.transform(bbox)
        return image, bbox
    
    def __len__(self):
        return len(self.data_path)<jupyter_output><empty_output><jupyter_text>DataLoader 생성<jupyter_code>train_path = glob.glob(data_path + '/train/*.jpg')
test_path = glob.glob(data_path + '/test/*.jpg')
mean = np.array([0.485, 0.456, 0.406])
std = np.array([0.229, 0.224, 0.225])

transform = transforms.Compose([
    #transforms.Resize((1050,753)), 
    transforms.ToTensor(),
    transforms.Normalize(mean=mean.tolist(), std=std.tolist())
])

trainset = MyDataset(data_path=train_path, 
                     bbox_frame=bbox_frame, 
                     transform=transform)
testset = MyDataset(data_path=test_path, 
                     bbox_frame=bbox_frame, 
                     transform=transform)

trainloader = torch.utils.data.DataLoader(trainset, batch_size=2, shuffle=True)
testloader = torch.utils.data.DataLoader(testset, batch_size=1, shuffle=False)

example_train = next(iter(trainloader))
example_test = next(iter(testloader))<jupyter_output><empty_output><jupyter_text>data 시각화<jupyter_code>from torchvision.utils import make_grid
from matplotlib.patches import Rectangle

image_ex = example_train[0]
bbox_ex = example_train[1]

# Unnomalization
image_temp = torch.zeros_like(image_ex)
for i, im in enumerate(image_ex):
    image_temp[i] = transforms.Normalize(mean=(-mean/std).tolist(), std=(1/std).tolist())(im)
    
grid = make_grid(image_temp)
grid_numpy = np.array(grid).transpose(1,2,0)

plt.figure(figsize=(10,10))
plt.imshow(grid_numpy)

#plt.gca().add_patch(Rectangle((bbox_ex[0][0][0], bbox_ex[0][0][1]), 50, 50, linewidth=1, edgecolor='r'))<jupyter_output><empty_output><jupyter_text>bbox 확인<jupyter_code>plt.figure(figsize = (10,10))

for i, (img, bbox) in enumerate(zip(image_temp, bbox_ex)):
    plt.subplot(2,2,i+1)
    plt.imshow(np.array(img).transpose(1,2,0))
    plt.gca().add_patch(Rectangle((bbox[0][0], bbox[0][1]), bbox[0][2], bbox[0][3], linewidth=3, edgecolor='r', facecolor='none'))<jupyter_output><empty_output><jupyter_text>Modeling우선 예시로 Faster-RCNN을 이용<jupyter_code>import torchvision
from torchvision.models.detection import fasterrcnn_resnet50_fpn
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor
model = fasterrcnn_resnet50_fpn(pretrained=True)
#in_features = model.roi_heads.box_predictor.cls_score.in_features
#model.roi_heads.box_predictor = FastRCNNPredictor(in_features)

model<jupyter_output><empty_output><jupyter_text>model forward test<jupyter_code>model.eval()

images, boxes = next(iter(trainloader))
labels = torch.zeros((4, 1))
print(images.shape, boxes.shape, labels.shape)

x = [images[0]]
predictions = model(x)
predictions
### plt.figure(figsize=(10,10))
image_temp = torch.zeros_like(images[0])
image_temp = transforms.Normalize(mean=(-mean/std).tolist(), std=(1/std).tolist())(images[0])
    
plt.imshow(np.array(image_temp).transpose(1,2,0))

for i, s in enumerate(predictions[0]['scores']):
    if s > 0:
        b = predictions[0]['boxes'][i]
        x = b[0]
        y = b[1]
        w = b[2]-b[0]
        h = b[3]-b[1]
        plt.gca().add_patch(Rectangle((x,y), w, h, linewidth=1, edgecolor='r', facecolor='none'))<jupyter_output><empty_output><jupyter_text>Training from torchvision.models.detection.rpn import AnchorGeneratorfrom torchvision.models.detection import FasterRCNNfrom torchvision.models import mobilenet_v2backbone = mobilenet_v2(pretrained=True).featuresbackbone.out_channels = 1280anchor_generator = AnchorGenerator(sizes=((32, 64, 128, 256, 512)),                                   aspect_ratios=((0.5, 1.0, 2.0)))roi_pooler = torchvision.ops.MultiScaleRoIAlign(featmap_names=[0],                                                output_size=7,                                                sampling_ratio=2)model = FasterRCNN(backbone,                   num_classes=2,                   rpn_anchor_generator=anchor_generator,                   box_roi_pool=roi_pooler)model.eval()<jupyter_code>image_path = 'data/image/face_example4.jpeg'
img = Image.open('data/image/face_example4.jpeg')
plt.imshow(img)
print(img.size)
model = MTCNN(
    image_size = 200,
    margin = 100,
    keep_all = True
)
model(img)
output = model(img)
output.shape
plt.figure(figsize = (10,10))

for i in range(output.shape[0]):
    plt.subplot(2,2,i+1)
    plt.imshow(output[i].permute(1,2,0))
plt.imshow(output.permute(1,2,0))

help(MTCNN)
model = MTCNN()
plt.imshow(np.array(model(img).permute(1,2,0)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13988175221714594, 'fraction_numerical': 0.03238376780435367, 'mean_word_length': 4.263790664780764, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '551252', 'license': 'MIT', 'n_tokens_mistral': 3043, 'n_tokens_neox': 2968, 'n_words': 457, 'path': 'Car_License_Recognition_DeepLearning.ipynb', 'repo_name': 'gjustin40/gjustin40_blog', 'size': '7442'}