<jupyter_start><jupyter_text>Imports<jupyter_code>import wandb
import os
import multiprocessing
import rasterio
import tensorflow as tf
from glob import glob
import pickle
import numpy as np
import os
from tensorflow.keras.models import *
from tensorflow.keras.layers import *
from tensorflow.keras import layers
from tensorflow_addons.metrics import F1Score, HammingLoss
from tensorflow_addons.losses import SigmoidFocalCrossEntropy

from tensorflow import keras
import pandas as pd
import matplotlib.pyplot as plt
import random
import time
import boto3
import io
from test_generator import TestGenerator
np.set_printoptions(suppress=True)
np.set_printoptions(precision=6)
print(tf.__version__)<jupyter_output>2.4.1<jupyter_text>Evaluation  Variables definition<jupyter_code># label_list = ["Habitation", "Industrial_agriculture", "Mining", "Rainforest", "River", "Roads", "Savannah", 
#               "Shifting_cultivation", "Water"]

new_label_list = ["Industrial_agriculture","ISL","Mining","Roads","Shifting_cultivation"]

# 1:"Industrial_agriculture"
# 2:"ISL"
# 3:"Mining"
# 4:"Roads"
# 5:"Shifting_cultivation"


numclasses = len(new_label_list)
bands_all = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16, 17, 18]
bands_RGBNIRN = [2, 3, 4, 8, 18]
# input_shape_all_bands= (100,100,len(bands_all))
input_shape_RGBNIRN = (100,100,len(bands_RGBNIRN))<jupyter_output><empty_output><jupyter_text>Define model (same architecture as the one used for the training)<jupyter_code>def define_model(numclasses,input_shape,starting_checkpoint=None,lcl_chkpt_dir=None):
        # parameters for CNN
        input_tensor = Input(shape=input_shape)

        # introduce a additional layer to get from bands to 3 input channels
        input_tensor = Conv2D(3, (1, 1))(input_tensor)

        base_model_resnet50 = keras.applications.ResNet50(include_top=False,
                                  weights='imagenet',
                                  input_shape=(100, 100, 3))
        base_model = keras.applications.ResNet50(include_top=False,
                         weights=None,
                         input_tensor=input_tensor)

        for i, layer in enumerate(base_model_resnet50.layers):
            # we must skip input layer, which has no weights
            if i == 0:
                continue
            base_model.layers[i+1].set_weights(layer.get_weights())

        # add a global spatial average pooling layer
        top_model = base_model.output
        top_model = GlobalAveragePooling2D()(top_model)

        # let's add a fully-connected layer
        top_model = Dense(2048, activation='relu')(top_model)
        top_model = Dense(2048, activation='relu')(top_model)
        # and a logistic layer
        predictions = Dense(numclasses, activation='sigmoid')(top_model)

        # this is the model we will train
        model = Model(inputs=base_model.input, outputs=predictions)
    #     model = model.layers[-1].bias.assign([0.0]) # WIP getting an error ValueError: Cannot assign to variable dense_8/bias:0 due to variable shape (10,) and value shape (1,) are incompatible

#         model.summary()
        if starting_checkpoint:
            s3 = boto3.resource('s3')
            chkpt_name = lcl_chkpt_dir + '/' + 'start_chkpt.h5'
            s3.download_file('canopy-production-ml-output', starting_checkpoint, chkpt_name)
            model.load_weights(chkpt_name)
    
        return model
# model_all_bands = define_model(numclasses, input_shape_all_bands)

# # Set the metrics we want to use for the evaluation
# model_all_bands.compile(loss=SigmoidFocalCrossEntropy(), # https://www.tensorflow.org/addons/api_docs/python/tfa/losses/SigmoidFocalCrossEntropy
#                   optimizer=keras.optimizers.Adam(0.001),
#                   metrics=[tf.metrics.BinaryAccuracy(name='accuracy'), 
#                            tf.keras.metrics.Precision(name='precision'), # Computes the precision of the predictions with respect to the labels.
#                            tf.keras.metrics.Recall(name='recall'), # Computes the recall of the predictions with respect to the labels.
#                            F1Score(num_classes=numclasses, name="f1_score") # https://www.tensorflow.org/addons/api_docs/python/tfa/metrics/F1Score
#                            ]
#                  )

model_RGBNIRN = define_model(numclasses, input_shape_RGBNIRN)

# Set the metrics we want to use for the evaluation
model_RGBNIRN.compile(loss=SigmoidFocalCrossEntropy(), # https://www.tensorflow.org/addons/api_docs/python/tfa/losses/SigmoidFocalCrossEntropy
                  optimizer=keras.optimizers.Adam(0.001),
                  metrics=[tf.metrics.BinaryAccuracy(name='accuracy'), 
                           tf.keras.metrics.Precision(name='precision'), # Computes the precision of the predictions with respect to the labels.
                           tf.keras.metrics.Recall(name='recall'), # Computes the recall of the predictions with respect to the labels.
                           F1Score(num_classes=numclasses, name="f1_score") # https://www.tensorflow.org/addons/api_docs/python/tfa/metrics/F1Score
                           ]
                 )<jupyter_output><empty_output><jupyter_text>Get the pre-trained weights from s3 (download them locally and then use the path in load_weights)<jupyter_code># KEY = "ckpt/pc-tf-custom-container-test-job-all-ban-2021-03-15-22-50-21-572/model_resnet_epoch_4.h5"

# s3 = boto3.resource('s3')
# s3.Bucket('canopy-production-ml-output').download_file(KEY, 'model_weights_all_bands.h5')

s3_uri = "s3://canopy-production-ml-output/ckpt/pc-tf-custom-container-test-job-RGBNIRN-2021-03-29-22-16-01-163/model_resnet_epoch_11.h5"

KEY = "/".join(s3_uri.split("/")[3:])
# KEY = "ckpt/pc-tf-custom-container-test-job-RGBNIRN-2021-03-27-02-29-15-006/model_resnet_epoch_31.h5"
s3 = boto3.resource('s3')
s3.Bucket('canopy-production-ml-output').download_file(KEY, 'model_weights_RGBNIRN.h5')
# model_all_bands.load_weights("model_weights_all_bands.h5")
model_RGBNIRN.load_weights("model_weights_RGBNIRN.h5")<jupyter_output><empty_output><jupyter_text>Test Set generator<jupyter_code>batch_size = 1
# csv_path = "labels_test.csv" 
# new_csv_path = "new_test_labels_v2.csv" 
new_csv_path_one_third = "new_test_labels_v3_one_third.csv"

# gen_all_bands = TestGenerator(label_file_path_test=csv_path, 
#                          bucket_name='canopy-production-ml',
#                          label_mapping_path="labels.json",
#                          data_extension_type='.tif',
#                          bands=bands_all,
#                          test_data_shape=input_shape_all_bands,
#                          test_data_batch_size=batch_size,
#                          enable_data_prefetch=False,
#                          data_prefetch_size=tf.data.experimental.AUTOTUNE,
#                          num_parallel_calls=2 * multiprocessing.cpu_count(),
#                          output_shape=(tf.float32, tf.float32))

gen_RGBNIRN = TestGenerator(label_file_path_test=new_csv_path_one_third,
                         bucket_name='canopy-production-ml',
                         label_mapping_path="new_labels.json",
                         data_extension_type='.tif',
                         bands=bands_RGBNIRN,
                         test_data_shape=input_shape_RGBNIRN,
                         test_data_batch_size=batch_size,
                         enable_data_prefetch=False,
                         data_prefetch_size=tf.data.experimental.AUTOTUNE,
                         num_parallel_calls=2 * multiprocessing.cpu_count(),
                         output_shape=(tf.float32, tf.float32))
# for img, label in gen_all_bands.test_dataset:
#     print(img.shape)
#     break<jupyter_output><empty_output><jupyter_text>Run Evaluation on the test set<jupyter_code># loss_all_bands, accuracy_all_bands, precision_all_bands, recall_all_bands, f1score_all_bands = model_all_bands.evaluate(gen_all_bands.test_dataset) 
loss_RGBNIRN, accuracy_RGBNIRN, precision_RGBNIRN, recall_RGBNIRN, f1score_RGBNIRN = model_RGBNIRN.evaluate(gen_RGBNIRN.test_dataset)<jupyter_output>97/9536 [..............................] - ETA: 38:28 - loss: 5.1684 - accuracy: 0.8202 - precision: 0.1019 - recall: 0.9199 - f1_score: 0.0684<jupyter_text>Predict on one image<jupyter_code>s3 = boto3.resource('s3')
# TODO test on entire test dataset
BUCKET_NAME = 'canopy-production-ml'
obj = s3.Object(BUCKET_NAME, "chips/cloudfree-merge-polygons/split/test/100/100_1000_1000.tif")
obj_bytes = io.BytesIO(obj.get()['Body'].read())
with rasterio.open(obj_bytes) as src:
    img_test = np.transpose(src.read(bands_RGBNIRN), (1, 2, 0))
print(img_test.shape)

predictions = model_RGBNIRN.predict(np.array([img_test]))

threshold = .2  
argsort_list = list(np.argsort(np.array(predictions[0])))
argsort_list.reverse()
pred_labels = []
for top_matches in argsort_list:
    if predictions[0][top_matches] > threshold:
        pred_labels.append(new_label_list[top_matches])

print("This chip was predicted to belong to these classes:")
print(pred_labels)

# print("correct label")

# def get_label(chip_uri,csv_path=csv_path):
    
#     df1 = pd.read_csv(csv_path)
#     df2 = pd.read_csv("/cb_feature_detection/data-prep/local_multi_label.csv")
    
#     return df2[df2["labels"].contains(chip_uri)]
    
# get_label("100/100_1000_1000.tif")<jupyter_output><empty_output><jupyter_text>Prediction on the test set<jupyter_code># Predict on all chips in test set
# y_pred_all_bands = model_all_bands.predict(gen_all_bands.test_dataset, verbose=1)
y_pred_RGBNIRN = model_RGBNIRN.predict(gen_RGBNIRN.test_dataset, verbose=1)
# print(y_pred_all_bands[0])
print(y_pred_RGBNIRN[0])
csv_path = "new_test_labels_v2.csv"

# Get true labels from csv file 
y_true_2 = pd.read_csv(new_csv_path_one_third).drop('paths', 1)
y_true = np.array([y_true_2.loc[i].to_list() for i in range(len(y_true_2))])
print(y_true)
def compare_ytrue_ypred(y_true, y_pred,threshold=0.2):
    for idx in range(len(y_pred)):
        argsort_list = list(np.argsort(np.array(y_pred[idx])))
        argsort_list.reverse()
        true_classes_list = [ i for i in range(len(y_true[idx])) if y_true[idx][i] == 1 ]
        print(f"\n{idx}) True labels:")
        print([label_list[index] for index in true_classes_list])
        pred_labels = []
        for top_matches in argsort_list:
            if y_pred[idx][top_matches] > threshold:
                pred_labels.append(label_list[top_matches])
        print("Predicted labels:")
        print(pred_labels)
#         break
# compare_ytrue_ypred(y_true, y_pred,threshold=0.2)<jupyter_output><empty_output><jupyter_text>Confusion matrices<jupyter_code>y_pred_RGBNIRN
threshold = .5

# y_pred_all_bands_threshold = np.where(y_pred_all_bands > threshold, 1, 0)
y_pred_RGBNIRN_threshold = np.where(y_pred_RGBNIRN > threshold, 1, 0)
y_pred_RGBNIRN_threshold
from sklearn.metrics import multilabel_confusion_matrix, ConfusionMatrixDisplay
# https://scikit-learn.org/stable/modules/generated/sklearn.metrics.multilabel_confusion_matrix.html
# matrices_all_bands = multilabel_confusion_matrix(y_true, y_pred_all_bands_threshold)
matrices_RGBNIRN = multilabel_confusion_matrix(y_true, y_pred_RGBNIRN_threshold)
# if no label x on chip i then label = 0 and predicted should = 0 otherwise if label predicted = 1 then it is wrongly classifiying the label (FP)
# if label x on chip i then label = 1 and predicted should = 1 otherwise if label predicted = 0 then it is missing the label classification (FN)
for i in range(len(new_label_list)):
#     cmd = ConfusionMatrixDisplay(matrices_all_bands[i], display_labels=np.unique(y_true)).plot()
#     plt.title(f'Confusion Matrix for label {label_list[i]} - All bands')
#     plt.show()    
    cmd = ConfusionMatrixDisplay(matrices_RGBNIRN[i], display_labels=np.unique(y_true)).plot()
    plt.title(f'Confusion Matrix for label {new_label_list[i]} - RGBNIRN')
    plt.show()
KEY = "ckpt/pc-tf-custom-container-test-job-all-ban-2021-03-15-22-50-21-572/model_resnet_epoch_15.h5"
s3 = boto3.resource('s3')
s3.Bucket('canopy-production-ml-output').download_file(KEY, 'model_weights_all_bands.h5')
model_all_bands.load_weights("model_weights_all_bands.h5")
y_pred_all_bands_2 = model_all_bands.predict(gen_all_bands.test_dataset, verbose=1)
y_pred_all_bands_threshold_2 = np.where(y_pred_all_bands_2 > threshold, 1, 0)
y_pred_all_bands_threshold_2
from sklearn.metrics import multilabel_confusion_matrix, ConfusionMatrixDisplay
# https://scikit-learn.org/stable/modules/generated/sklearn.metrics.multilabel_confusion_matrix.html
matrices_all_bands_2 = multilabel_confusion_matrix(y_true, y_pred_all_bands_threshold_2)
for i in range(len(label_list)):
    cmd = ConfusionMatrixDisplay(matrices_all_bands_2[i], display_labels=np.unique(y_true)).plot()
    plt.title(f'Confusion Matrix for label {label_list[i]} - All bands')
    plt.show()    
loss_all_bands, accuracy_all_bands, precision_all_bands, recall_all_bands, f1score_all_bands = model_all_bands.evaluate(gen_all_bands.test_dataset)<jupyter_output><empty_output><jupyter_text>Sandbox<jupyter_code>df = pd.read_csv("labels_test.csv")
df["2"].value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10933716707021791, 'fraction_numerical': 0.025726392251815982, 'mean_word_length': 3.7457809694793536, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 44, 'https://': 6, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '130895', 'license': 'MIT', 'n_tokens_mistral': 4696, 'n_tokens_neox': 4442, 'n_words': 745, 'path': 'model-development/_archive/evaluation_Zhenya_epoch_4.ipynb', 'repo_name': 'Project-Canopy/SSRC_New_Model_Development', 'size': '13216'}