Finalized evaluation step, which now works. Ready to merge into master

dvc.lock

@@ -20,8 +20,8 @@ train:
   cmd: python3 src/code/training.py src/data/processed/train
   deps:
   - path: src/code/custom_data_loading.py
-    md5: e04501397e3333bcec07eb8ef633b234
-    size: 2172
+    md5: c94ea029ed76ca94bb1ad4c1655e5e68
+    size: 1916
   - path: src/code/params.yml
     md5: 2263ca2167c1bb4b0f53a9aedb5f238e
     size: 217
@@ -34,12 +34,38 @@ train:
     nfiles: 1590
   outs:
   - path: logs/train_metrics.csv
-    md5: 3504c0db314c5d285f3254f198a85a09
-    size: 103963
+    md5: 437a06e6c6c5b4f6eec5e546c1ce6930
+    size: 103916
   - path: logs/train_params.yml
-    md5: a2663346b17d042e5da48991fcf62771
+    md5: e06e92ac0f3ac1d367c22a10c28cccf9
     size: 886
   - path: src/models/
-    md5: 8969d16fb5b908cbfb6b54dffbf20efc.dir
-    size: 494926837
+    md5: fab42526c433987e0e6370db31a1869d.dir
+    size: 494927196
     nfiles: 1
+eval:
+  cmd: python3 src/code/eval.py src/data/processed/test
+  deps:
+  - path: src/code/custom_data_loading.py
+    md5: c94ea029ed76ca94bb1ad4c1655e5e68
+    size: 1916
+  - path: src/code/eval.py
+    md5: fcc66ed80bb4466ab0438f556acd125c
+    size: 1775
+  - path: src/code/eval_metric_calculation.py
+    md5: 2fc866e1107042a996087d5716d44bf0
+    size: 2999
+  - path: src/code/params.yml
+    md5: 2263ca2167c1bb4b0f53a9aedb5f238e
+    size: 217
+  - path: src/data/processed/test
+    md5: bcccd66f3f561b53ba97c89a558c08a0.dir
+    size: 88596370
+    nfiles: 1308
+  - path: src/models/model.pth
+    md5: 2fd77305fd779eefd11e307ee3f201d7
+    size: 494927196
+  outs:
+  - path: logs/test_metrics.csv
+    md5: 0add355c58eb4dfa1ae7e28e47750d33
+    size: 340

logs/test_metrics.csv

@@ -0,0 +1,10 @@
+Name,Value,Timestamp,Step
+"a1",0.056999333,1613824849186,1
+"a2",0.118539445,1613824849186,1
+"a3",0.19929159,1613824849186,1
+"abs_rel",2.5860002,1613824849186,1
+"sq_rel",15.912783,1613824849186,1
+"rmse",5.257741,1613824849186,1
+"rmse_log",1.2291939,1613824849186,1
+"log10",0.49469143,1613824849186,1
+"silog",43.5198,1613824849186,1

logs/train_params.yml

@@ -1,5 +1,5 @@
 DAGsHubLogger: true
-Learner: <fastai.learner.Learner object at 0x7f949b510cc0>
+Learner: <fastai.learner.Learner object at 0x7f051ecfcac8>
 ParamScheduler: true
 ProgressCallback: true
 Recorder: {add_time: true, train_metrics: false, valid_metrics: true}

requirements.txt

@@ -6,4 +6,5 @@ opencv-python==4.4.0.42
 tqdm==4.52.0
 numpy==1.19.4
 scikit-learn==0.23.2
-dagshub==0.1.5
+dagshub==0.1.5
+tables==3.6.1

src/.gitignore

@@ -1 +1,2 @@
 /models
+/eval

src/code/custom_data_loading.py

@@ -8,8 +8,7 @@ from fastai.vision.all import \
     PILImageBW, \
     RandomSplitter, \
     Path, \
-    get_files, \
-    L
+    get_files
 
 
 class ImageImageDataLoaders(DataLoaders):
@@ -35,7 +34,7 @@ def get_y_fn(x):
     return y
 
 
-def create_data(data_path, is_test=False):
+def create_data(data_path):
     with open(r"./src/code/params.yml") as f:
         params = yaml.safe_load(f)
 
@@ -49,10 +48,4 @@ def create_data(data_path, is_test=False):
                                                     filenames=filenames,
                                                     label_func=get_y_fn)
 
-    if is_test:
-        filenames = get_files(Path(data_path), extensions='.jpg')
-        test_files = L([Path(i) for i in filenames])
-        test_dl = dataset.test_dl(test_files, with_labels=True)
-        return dataset, test_dl
-
     return dataset

src/code/eval.py

@@ -1,9 +1,12 @@
 import sys
 import yaml
-from fastai.vision.all import unet_learner, Path, resnet34, MSELossFlat, tuplify
+import torch
+from torchvision import transforms
+from fastai.vision.all import unet_learner, Path, resnet34, MSELossFlat, get_files, L, tuplify
 from custom_data_loading import create_data
 from eval_metric_calculation import compute_eval_metrics
 from dagshub import dagshub_logger
+from tqdm import tqdm
 
 
 if __name__ == "__main__":
@@ -15,7 +18,7 @@ if __name__ == "__main__":
         params = yaml.safe_load(f)
 
     data_path = Path(sys.argv[1])
-    data, test_dl = create_data(data_path, is_test=True)
+    data = create_data(data_path)
 
     arch = {'resnet34': resnet34}
     loss = {'MSELossFlat': MSELossFlat()}
@@ -28,20 +31,16 @@ if __name__ == "__main__":
                            model_dir='models')
     learner = learner.load('model')
 
-    print("Running model on test data...")
-    inputs = learner.get_preds(dl=test_dl,
-                               with_input=True,
-                               with_decoded=True,
-                               save_preds=Path('src/eval/preds'),
-                               save_targs=Path('src/eval/targs'))[0]
-    # FastAI magic to retrieve image values
-    print("Decoding test data...")
-    inputs = (inputs,)
-    decoded_predictions = learner.dls.decode(inputs + tuplify(decoded))[1]
-    decoded_targets = learner.dls.decode(inputs + tuplify(targets))[1]
+    filenames = get_files(Path(data_path), extensions='.jpg')
+    test_files = L([Path(i) for i in filenames])
+
+    for sample in tqdm(test_files.items, desc="Predicting on test images", total=len(test_files.items)):
+        pred = learner.predict(sample)[0]
+        pred = transforms.ToPILImage()(pred[:, :, :].type(torch.FloatTensor)).convert('L')
+        pred.save("src/eval/" + str(sample.stem) + "_pred.png")
 
     print("Calculating metrics...")
-    metrics = compute_eval_metrics(decoded_targets.numpy(), decoded_predictions.numpy())
+    metrics = compute_eval_metrics(test_files)
 
     with dagshub_logger(
             metrics_path="logs/test_metrics.csv",

src/code/eval_metric_calculation.py

@@ -1,5 +1,6 @@
 import numpy as np
-
+from PIL import Image
+from tqdm import tqdm
 
 def compute_errors(target, prediction):
     thresh = np.maximum((target / prediction), (prediction / target))
@@ -24,14 +25,11 @@ def compute_errors(target, prediction):
     return a1, a2, a3, abs_rel, sq_rel, rmse, rmse_log, silog, log_10
 
 
-def compute_eval_metrics(targets, predictions):
-    targets = targets / 25.0
-    predictions = predictions / 25.0
-
+def compute_eval_metrics(test_files):
     min_depth_eval = 1e-3
     max_depth_eval = 10
 
-    num_samples = predictions.shape[0]
+    num_samples = len(test_files)
 
     a1 = np.zeros(num_samples, np.float32)
     a2 = np.zeros(num_samples, np.float32)
@@ -43,21 +41,31 @@ def compute_eval_metrics(targets, predictions):
     silog = np.zeros(num_samples, np.float32)
     log10 = np.zeros(num_samples, np.float32)
 
-    for i in range(num_samples):
-        target_depth = targets[i]
-        prediction_depth = predictions[i]
+    for i in tqdm(range(num_samples), desc="Calculating metrics for test data", total=num_samples):
+        sample_path = test_files[i]
+        target_path = str(sample_path.parent/(sample_path.stem + "_depth.png"))
+        pred_path = "src/eval/" + str(sample_path.stem) + "_pred.png"
+
+        target_image = Image.open(target_path)
+        pred_image = Image.open(pred_path)
+
+        target = np.asarray(target_image)
+        pred = np.asarray(pred_image)
+
+        target = target / 25.0
+        pred = pred / 25.0
 
-        prediction_depth[prediction_depth < min_depth_eval] = min_depth_eval
-        prediction_depth[prediction_depth > max_depth_eval] = max_depth_eval
-        prediction_depth[np.isinf(prediction_depth)] = max_depth_eval
+        pred[pred < min_depth_eval] = min_depth_eval
+        pred[pred > max_depth_eval] = max_depth_eval
+        pred[np.isinf(pred)] = max_depth_eval
 
-        target_depth[np.isinf(target_depth)] = 0
-        target_depth[np.isnan(target_depth)] = 0
+        target[np.isinf(target)] = 0
+        target[np.isnan(target)] = 0
 
-        valid_mask = np.logical_and(target_depth > min_depth_eval, target_depth < max_depth_eval)
+        valid_mask = np.logical_and(target > min_depth_eval, target < max_depth_eval)
 
         a1[i], a2[i], a3[i], abs_rel[i], sq_rel[i], rmse[i], rmse_log[i], silog[i], log10[i] = \
-            compute_errors(target_depth[valid_mask], prediction_depth[valid_mask])
+            compute_errors(target[valid_mask], pred[valid_mask])
 
     print("{:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}".format(
         'd1', 'd2', 'd3', 'AbsRel', 'SqRel', 'RMSE', 'RMSElog', 'SILog', 'log10'))