refer/inference.py

import os
import cv2
import numpy as np
import torch
import torch.backends.cudnn as cudnn
from models_refer.model import EVPRefer
from args import get_parser
import glob
import utils
import torchvision.transforms as transforms
from PIL import Image
import torch.nn.functional as F
from transformers import CLIPTokenizer


def main():
    parser = get_parser()
    parser.add_argument('--img_path',  type=str)
    parser.add_argument('--prompt',  type=str)
    args = parser.parse_args()

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-large-patch14")
    model = EVPRefer(sd_path='../checkpoints/v1-5-pruned-emaonly.ckpt')
    cudnn.benchmark = True
    model.to(device)
    model_weight = torch.load(args.resume)['model']
    if 'module' in next(iter(model_weight.items()))[0]:
        model_weight = OrderedDict((k[7:], v) for k, v in model_weight.items())
    model.load_state_dict(model_weight, strict=False)
    model.eval()
    
    img_path = args.img_path
    
    image = cv2.imread(img_path)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image_t = transforms.ToTensor()(image).unsqueeze(0).to(device)
    image_t = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])(image_t)
    shape = image_t.shape
    image_t = torch.nn.functional.interpolate(image_t, (512,512), mode='bilinear', align_corners=True)
    input_ids = tokenizer(text=args.prompt, truncation=True, max_length=args.token_length, return_length=True,
            return_overflowing_tokens=False, padding="max_length", return_tensors="pt")['input_ids'].to(device)
    
    with torch.no_grad():
        pred = model(image_t, input_ids)
    
    pred = torch.nn.functional.interpolate(pred, shape[2:], mode='bilinear', align_corners=True)
    output_mask = pred.cpu().argmax(1).data.numpy().squeeze()
    
    alpha = 0.65
    image[output_mask == 0] = (image[output_mask == 0]*alpha).astype(np.uint8)
    contours, _ = cv2.findContours(output_mask.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    cv2.drawContours(image, contours, -1, (0, 255, 0), 2)

    Image.fromarray(image.astype(np.uint8)).save('res.png')

    return 0

if __name__ == '__main__':
    main()