<jupyter_start><jupyter_text>1. Load model (fastrcnn)<jupyter_code>def get_model(num_classes,image_mean,image_std):
    model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True,
                                                                image_mean=image_mean,
                                                                image_std=image_std)
    in_features = model.roi_heads.box_predictor.cls_score.in_features
    model.roi_heads.box_predictor = torchvision.models.detection.faster_rcnn.FastRCNNPredictor(in_features,num_classes)
    return model
fastercnn = get_model(10,image_mean=val_bdd_oia_dataset.mean,image_std=val_bdd_oia_dataset.std)
checkpoint = torch.load('saved_models/bdd100k_24.pth')
fastercnn.load_state_dict(checkpoint['model'])

# # Load version 1 
# from decision_generator_model import DecisionGenerator_v1
# decision_generator = DecisionGenerator_v1(fastercnn,device,batch_size)

# checkpoint = torch.load("saved_models/bdd_oia19.pth")
# decision_generator.load_state_dict(checkpoint["model"])


# Load version 2
from decision_generator_model import DecisionGenerator
decision_generator = DecisionGenerator(fastercnn,device,batch_size)

checkpoint = torch.load("saved_models/bdd_oia_sgd9.pth")
decision_generator.load_state_dict(checkpoint["model"])

decision_generator = decision_generator.to(device)
decision_generator.eval()<jupyter_output><empty_output><jupyter_text>2. Load transformer  3. Test<jupyter_code>sample = next(iter(val_loader))
len(sample)
sample = next(iter(val_loader))
images, labels = sample

with open('./ce 597 demo/readme.txt') as f:
    txt = f.readlines()
all_reasons = np.array([lab.strip()[:-1] for lab in txt[6:-1]])
all_actions = np.array(['move forward', 'stop/slow down', 'turn left', 'turn right'])

def imshow(inp, title=None):
    """Imshow for Tensor."""
    inp = inp.numpy().transpose((1, 2, 0))
    plt.imshow(inp)
    if title is not None:
        plt.title(title)
    plt.pause(0.001)  # pause a bit so that plots are updated

action_batch = [all_actions[l['action'].numpy()==1] for l in labels]
reason_batch = [all_reasons[r['reason'].numpy()==1] for r in labels]
out = torchvision.utils.make_grid(list(images))

predictions = decision_generator(list(image.to(device) for image in images))

action_predicted_batch = predictions['action'].detach().cpu().numpy()>0.5
reason_predicted_batch = predictions['reasons'].detach().cpu().numpy()>0.5

action_predicted_batch = [all_actions[l] for l in action_predicted_batch]
reason_predicted_batch = [all_reasons[r] for r in reason_predicted_batch]
plt.figure(figsize=(30,20))
imshow(out)

for a,b,ap,bp in zip(action_batch,reason_batch,action_predicted_batch,reason_predicted_batch):
    
    print("predicted actions: ", list(ap))
    print("target actions: ", list(a))

    print("predicted reasons: ", list(bp))
    print("target reasons: ", list(b))

    print()<jupyter_output><empty_output><jupyter_text>Transformer test<jupyter_code>from decision_generator_model import *

def get_encoder(model_name):
    if model_name == 'mobile_net':
        md = torchvision.models.mobilenet_v2(pretrained=True)
        encoder = nn.Sequential(*list(md.children())[:-1])
    elif model_name == 'resnet':
        md = torchvision.models.resnet50(pretrained=True)
        encoder = nn.Sequential(*list(md.children())[:-2])
    else:

        raise ValueError("unrecognized encoder model name: "+encoder_name)
    return encoder
from datasets.bdd_oia import BDD_OIA
image_dir = './data/bdd_oia/lastframe/data/'
label_dir = './data/bdd_oia/lastframe/labels/'
batch_size = 5
val_bdd_oia_dataset = BDD_OIA(image_dir,label_dir+'val_25k_images_actions.json',
                            label_dir+'val_25k_images_reasons.json',image_min_size=180)

val_loader = DataLoader(val_bdd_oia_dataset,shuffle=True,batch_size=batch_size,collate_fn=utils.collate_fn)
with open('./ce 597 demo/readme.txt') as f:
    txt = f.readlines()
all_reasons = np.array([lab.strip()[:-1] for lab in txt[6:-1]])
all_actions = np.array(['move forward', 'stop/slow down', 'turn left', 'turn right'])

encoder_name, encoder_dims = 'resnet',(2048,6,10)
model_name = 'whole_attention_resnet_39'
encoder = get_encoder(encoder_name)
decision_generator = DecisionGenerator_whole_attention(encoder,
                                                        encoder_dims=encoder_dims,
                                                        device=device)
decision_generator = decision_generator.to(device)
# print(count_parameters(decision_generator))
checkpoint = torch.load("/home/ai/Desktop/Jiqian work/work4/saved_models/whole_attention/%s.pth"%model_name)
decision_generator.load_state_dict(checkpoint["model"])
decision_generator.eval()
print("nb")
images, labels = next(iter(val_loader))

predictions = decision_generator(list(image.to(device) for image in images))
action_predicted_batch = predictions['action'].detach().cpu().numpy()>0.5
reason_predicted_batch = predictions['reasons'].detach().cpu().numpy()>0.5

action_batch = [all_actions[l['action'].numpy()==1] for l in labels]
reason_batch = [all_reasons[r['reason'].numpy()==1] for r in labels]
out = torchvision.utils.make_grid(list(images))

action_predicted_batch = [all_actions[l] for l in action_predicted_batch]
reason_predicted_batch = [all_reasons[r] for r in reason_predicted_batch]
img_names = [l['img_name'] for l in labels]
dst_names = ['./results/figures/' + str(i) + '.jpg' for i in range(len(img_names))]
from shutil import copyfile
[copyfile(src, dst) for src,dst in zip(img_names,dst_names)] 
inp = out.numpy().transpose((1, 2, 0))
plt.imshow(inp)
plt.savefig('./results/figures/images.png')
for a,b,ap,bp in zip(action_batch,reason_batch,action_predicted_batch,reason_predicted_batch):
    
    print("predicted actions: ", list(ap))
    print("target actions: ", list(a))

    print("predicted reasons: ", list(bp))
    print("target reasons: ", list(b))

    print()<jupyter_output>predicted actions:  ['stop/slow down']
target actions:  ['stop/slow down']
predicted reasons:  ['s_ob_ped: Stop/slow down - obstacle: person/pedestrain', 's_traffic_light: Stop/slow down - the traffic light']
target reasons:  ['s_ob_ped: Stop/slow down - obstacle: person/pedestrain']

predicted actions:  ['stop/slow down']
target actions:  ['stop/slow down']
predicted reasons:  ['s_traffic_light: Stop/slow down - the traffic light']
target reasons:  ['s_ob_car: Stop/slow down - obstacle: car', 's_traffic_light: Stop/slow down - the traffic light']

predicted actions:  ['move forward']
target actions:  ['stop/slow down']
predicted reasons:  ['f_follow_traffic: Forward - follow traffic', 'f_traffic_light: Forward - the traffic light is green']
target reasons:  ['s_traffic_light: Stop/slow down - the traffic light']

predicted actions:  ['move forward', 'turn right']
target actions:  ['move forward', 'turn right']
predicted reasons:  ['f_follow_traffic: Forward - follow traffic', 'f_traff[...]
{'fraction_non_alphanumeric': 0.13152220784821045, 'fraction_numerical': 0.011499209429351733, 'mean_word_length': 4.9725321888412015, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '527256', 'license': 'MIT', 'n_tokens_mistral': 2383, 'n_tokens_neox': 2288, 'n_words': 438, 'path': 'inference_decision_generator.ipynb', 'repo_name': 'JiqianDong/BDD-100-CV', 'size': '6957'}