<jupyter_start><jupyter_text>Importing Libraries<jupyter_code>from fastai.vision import *
from fastai.callbacks import *<jupyter_output><empty_output><jupyter_text>Downloading Dataset and Setting up Paths<jupyter_code>path = untar_data(URLs.CAMVID)
print(path)
path.ls()
imgs_path = path/'images'
masks_path = path/'labels'
valid_files_path = path/'valid.txt'
classes_path = path/'codes.txt'<jupyter_output><empty_output><jupyter_text>Exploring the Dataset<jupyter_code>img_filenames = get_image_files(imgs_path)
masks_filenames = get_image_files(masks_path)
print(len(img_filenames))
print(len(masks_filenames))
img_filenames[:3]
masks_filenames[:3]<jupyter_output><empty_output><jupyter_text>**Opening an Image and its corresponding Mask**<jupyter_code>img_file = open_image(imgs_path/img_filenames[0])
img_file.show(figsize = (5,5), title = 'demo input image')
mask = open_mask(masks_path/masks_filenames[0], convert_mode = "L")
mask.show(figsize = (5,5), title = 'demo mask image', alpha = 1)<jupyter_output><empty_output><jupyter_text>**Getting the Pixel Classes**<jupyter_code>classes = np.loadtxt(classes_path, dtype = str)
print(len(classes))
print(classes)<jupyter_output>32
['Animal' 'Archway' 'Bicyclist' 'Bridge' 'Building' 'Car' 'CartLuggagePram' 'Child' 'Column_Pole' 'Fence'
 'LaneMkgsDriv' 'LaneMkgsNonDriv' 'Misc_Text' 'MotorcycleScooter' 'OtherMoving' 'ParkingBlock' 'Pedestrian' 'Road'
 'RoadShoulder' 'Sidewalk' 'SignSymbol' 'Sky' 'SUVPickupTruck' 'TrafficCone' 'TrafficLight' 'Train' 'Tree' 'Truck_Bus'
 'Tunnel' 'VegetationMisc' 'Void' 'Wall']<jupyter_text>Training with Half the Input Image size **Function to map Input Image to Mask**<jupyter_code>get_label_mask_file = lambda x : masks_path/f'{x.stem}_P{x.suffix}'<jupyter_output><empty_output><jupyter_text>**Creating the Databunch**<jupyter_code>src = (SegmentationItemList.from_folder(imgs_path)
                           .split_by_fname_file(valid_files_path)
                           .label_from_func(get_label_mask_file, classes = classes))
# Halving the Input Image Size
input_img_size = np.array(img_file.shape[1:]) // 2   

batch_size = 4
data = (src.transform(get_transforms(), size = input_img_size, tfm_y = True)
           .databunch(bs = batch_size, num_workers = 0)
           .normalize(imagenet_stats))
data
data.show_batch(2, figsize = (7,7))<jupyter_output><empty_output><jupyter_text>**Creating the Learner**<jupyter_code>name_to_index_map = {name:index for index, name in enumerate(classes)}
void_index = name_to_index_map['Void']
'''We define a Custom Accuracy metric since accuracy is to be measured without taking into account the pixels labelled as 'Void'
as those pixels are semantically ambiguous and hence not assigned a class while labelling the Dataset'''


def camvid_accuracy(output, target):
    target = target.squeeze(1)
    m = target != void_index
    mean_acc = (output.argmax(dim=1)[m] == target[m]).float().mean()
    return mean_acc
learn = unet_learner(data, models.resnet34, metrics = camvid_accuracy).to_fp16()<jupyter_output><empty_output><jupyter_text>**Fitting the Learner**<jupyter_code>learn.fit_one_cycle(10)
learn.save('half-size-stage-1')<jupyter_output><empty_output><jupyter_text>**Finetuning the Learner**<jupyter_code>learn.unfreeze()
learn.lr_find()
learn.recorder.plot()
learning_rate = slice(1e-5,1e-4)
learn.fit_one_cycle(15, max_lr = learning_rate)
learn.save('half-size-stage-2')<jupyter_output><empty_output><jupyter_text>Training with the actual Input Image size **Creating the Databunch**<jupyter_code>input_img_size = np.array(img_file.shape[1:])

batch_size = 2
data = (src.transform(get_transforms(), size = input_img_size, tfm_y = True)
           .databunch(bs = batch_size, num_workers = 0)
           .normalize(imagenet_stats))
data<jupyter_output><empty_output><jupyter_text>**Creating the Learner**<jupyter_code>learn = unet_learner(data, models.resnet34, metrics = camvid_accuracy).to_fp16()
learn.load('half-size-stage-2');
learn.fit_one_cycle(10)
learn.save('full-size-stage-1')<jupyter_output><empty_output><jupyter_text>**Finetuning the Learner**<jupyter_code>learn.unfreeze()
learn.lr_find()
learn.recorder.plot()
learning_rate = slice(1e-5,1e-4)
learn.fit_one_cycle(15, max_lr = learning_rate, callbacks = SaveModelCallback(learn, name = 'full-size-stage-2'))<jupyter_output><empty_output><jupyter_text>Checking Output Predictions<jupyter_code>valid_loss_metrics = learn.validate()
print("Validation Loss: ", valid_loss_metrics[0])
print("Validation Accuracy: ", valid_loss_metrics[1].item() * 100)
learn.show_results(rows=2, figsize=(10,10))<jupyter_output><empty_output><jupyter_text>Exporting the Trained Model<jupyter_code>learn.export('final_trained_model.pkl')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13162321278385197, 'fraction_numerical': 0.013036164844407064, 'mean_word_length': 7.633393829401089, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '428890', 'license': 'MIT', 'n_tokens_mistral': 1807, 'n_tokens_neox': 1751, 'n_words': 285, 'path': 'CamVid-Image-Segmentation.ipynb', 'repo_name': 'rdutta1999/CamVid-Image-Segmentation', 'size': '4756'}