<jupyter_start><jupyter_text>Load a (frozen) Tensorflow model into memory.<jupyter_code># !!!!! MODIFY THIS !!!!!
model = 'russian_signs_ssd'

detection_graph = tf.Graph()
with detection_graph.as_default():
    od_graph_def = tf.GraphDef()
    with tf.gfile.GFile(model+'/fine_tuned_model/frozen_inference_graph.pb', 'rb') as fid:
        serialized_graph = fid.read()
        od_graph_def.ParseFromString(serialized_graph)
        tf.import_graph_def(od_graph_def, name='')<jupyter_output><empty_output><jupyter_text>utils<jupyter_code># !!!!! MODIFY THIS !!!!!
label_map = label_map_util.load_labelmap('russian_signs_data/label_map.pbtxt')
NUM_CLASSES= 198 #20, 198
categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=NUM_CLASSES, use_display_name=True)
category_index = label_map_util.create_category_index(categories)
def load_image_into_numpy_array(image):
    (im_width, im_height) = image.size
    return np.array(image.getdata()).reshape((im_height, im_width, 3)).astype(np.uint8)
def black_background_thumbnail(image, thumbnail_size):
    background = Image.new('RGB', thumbnail_size, "black")    
    source_image = image
    source_image.thumbnail(thumbnail_size)
    (w, h) = source_image.size
    background.paste(source_image, (int((thumbnail_size[0] -w) / 2), int((thumbnail_size[1] - h) / 2 )))
    return background<jupyter_output><empty_output><jupyter_text>Detection<jupyter_code>import time

def run_inference_for_single_image(image):
    image_expanded = np.expand_dims(image, 0)

    # Run inference
    #start = time.time()
    (boxes, scores, classes, num) = \
        sess.run([detection_boxes, detection_scores, detection_classes, num_detections], 
                 feed_dict={image_tensor: image_expanded})
    
    #end = time.time()
    #print("only session {}".format(end-start))

    # all outputs are float32 numpy arrays, so convert types as appropriate
    #output_dict['num_detections'] = int(output_dict['num_detections'][0])
    output_dict = {}
    output_dict['detection_classes'] =  np.squeeze(classes).astype(np.uint8)
    output_dict['detection_boxes'] =  np.squeeze(boxes)
    output_dict['detection_scores'] =  np.squeeze(scores)
    if 'detection_masks' in output_dict:
        output_dict['detection_masks'] = output_dict['detection_masks'][0]
        
    return output_dict<jupyter_output><empty_output><jupyter_text>classification<jupyter_code>from keras.models import Sequential
from keras.layers import Dense, Conv2D, MaxPooling2D, Dropout, Flatten
 
def get_model():
    model = Sequential()
    model.add(Conv2D(32, (3, 3), padding='same', activation='relu', input_shape=input_shape))
    model.add(Conv2D(32, (3, 3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
    model.add(Conv2D(64, (3, 3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
    model.add(Conv2D(64, (3, 3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Flatten())
    model.add(Dense(512, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(len(LABELS), activation='softmax'))
    return model<jupyter_output>Using TensorFlow backend.<jupyter_text>get data<jupyter_code>import os
os.chdir('../../../')

from russian_signs.get_data import get_data as get_data_russian
from swedish_signs.get_data import get_data as get_data_swedish

os.chdir('models/research/object_detection')

# !!!!! MODIFY THIS !!!!!
data = get_data_russian()
LABELS = data['LABELS']
all_images = data['images']
input_shape = data['input_shape']
images = np.random.choice(all_images, 2000)<jupyter_output><empty_output><jupyter_text>inspect ground truth and model predictions<jupyter_code>from keras.preprocessing.image import img_to_array


with detection_graph.as_default():
    with tf.Session() as sess:
        
        class_model = get_model()
        # !!! MODIFY THIS !!!!
        class_model.load_weights('../../../classify_croped_weights_russian')
        
        # Definite input and output Tensors for detection_graph
        image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
        # Each box represents a part of the image where a particular object was detected.
        detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0')
        # Each score represent how level of confidence for each of the objects.
        # Score is shown on the result image, together with the class label.
        detection_scores = detection_graph.get_tensor_by_name('detection_scores:0')
        detection_classes = detection_graph.get_tensor_by_name('detection_classes:0')
        num_detections = detection_graph.get_tensor_by_name('num_detections:0')
        
        for image in images:
            print(image['filename'])
            img = Image.open(image['filename'])
            image_np = load_image_into_numpy_array(img)

            copy_image = image_np.copy()
            copy_image_2 = image_np.copy()
            copy_img = img.copy()

            print("GROUND TRUTH")

            signs = image['signs']

            boxes = np.array([[sign['ymin'], sign['xmin'], sign['ymax'], sign['xmax']] for sign in signs])
            classes =  np.array([LABELS.index(sign['name'])+1 for sign in signs])
            scores = np.array([1.0 for sign in signs])

            vis_util.visualize_boxes_and_labels_on_image_array(
              image_np,
              boxes,
              classes,
              scores,
              category_index,
              use_normalized_coordinates=True,
              line_thickness=3)

            plt.figure(figsize=(20,20))
            plt.imshow(image_np)
            plt.show()

            print("MODEL PREDICTIONS RAW")
            output_dict = run_inference_for_single_image(copy_image)
            boxes = output_dict['detection_boxes']
            classes = output_dict['detection_classes']
            scores = output_dict['detection_scores']

            vis_util.visualize_boxes_and_labels_on_image_array(
              copy_image,
              boxes,
              classes,
              scores,
              category_index,
              use_normalized_coordinates=True,
              line_thickness=3)

            plt.figure(figsize=(20,20))
            plt.imshow(copy_image)
            plt.show()
            
            print("MODEL PREDICTIONS CLASS")
            output_dict = run_inference_for_single_image(copy_image_2)
            boxes = output_dict['detection_boxes']
            classes = output_dict['detection_classes']
            scores = output_dict['detection_scores']
            
            classes = []

            w, h = copy_img.size

            for i, box in enumerate(boxes):

                left = box[1] * w
                top = box[0] * h
                right = box[3] * w
                bottom = box[2] * h

                crop_img = copy_img.crop((left, top, right, bottom))
                crop_img = crop_img.resize((input_shape[0], input_shape[1]))
                crop_img = img_to_array(crop_img)
                pred = class_model.predict(np.expand_dims(crop_img, axis=0))

                classes.append(np.argmax(pred[0])+1)
            classes = np.array(classes)

            vis_util.visualize_boxes_and_labels_on_image_array(
              copy_image_2,
              boxes,
              classes,
              scores,
              category_index,
              use_normalized_coordinates=True,
              line_thickness=3)

            plt.figure(figsize=(20,20))
            plt.imshow(copy_image_2)
            plt.show()<jupyter_output><empty_output><jupyter_text>calculate mAP<jupyter_code>len(images)
from keras.preprocessing.image import img_to_array

def get_preds(image_dict, use_classification_croped=False, box_scores_thres=0.6):
    
    image = Image.open(image_dict['filename'])
    #image = black_background_thumbnail(image, (1280, 960))
    image_np = load_image_into_numpy_array(image)
    image_np_expanded = np.expand_dims(image_np, axis=0)
    
    #start = time.time()
    output_dict = run_inference_for_single_image(image_np)
    #end = time.time()
    
    #print("all {}".format(end-start))
    
    inds = output_dict['detection_scores'] > box_scores_thres
    boxes = output_dict['detection_boxes'][inds]
    scores = output_dict['detection_scores'][inds]
    classes = output_dict['detection_classes'][inds]

    if use_classification_croped:
        classes = []

        w, h = image.size

        for i, box in enumerate(boxes):
            
            left = box[1] * w
            top = box[0] * h
            right = box[3] * w
            bottom = box[2] * h

            crop_img = image.crop((left, top, right, bottom))
            crop_img = crop_img.resize((input_shape[0], input_shape[1]))
            crop_img = img_to_array(crop_img)
            pred = class_model.predict(np.expand_dims(crop_img, axis=0))

            classes.append(np.argmax(pred[0])+1)
        classes = np.array(classes)
                
    return boxes, scores, classes     
all_detections     = [[None for i in range(len(LABELS))] for j in range(len(images))]
all_annotations    = [[None for i in range(len(LABELS))] for j in range(len(images))]

num_preds = 0

import time

start = time.time()
with detection_graph.as_default():
    with tf.Session() as sess:
        class_model = get_model()
        # !!! MODIFY THIS !!!!
        class_model.load_weights('../../../classify_croped_weights_russian')
        # Definite input and output Tensors for detection_graph
        image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
        # Each box represents a part of the image where a particular object was detected.
        detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0')
        # Each score represent how level of confidence for each of the objects.
        # Score is shown on the result image, together with the class label.
        detection_scores = detection_graph.get_tensor_by_name('detection_scores:0')
        detection_classes = detection_graph.get_tensor_by_name('detection_classes:0')
        num_detections = detection_graph.get_tensor_by_name('num_detections:0')
        
        for i, image_dict in enumerate(images):
            boxes, scores, classes = get_preds(image_dict, use_classification_croped=True)
            score_sort = np.argsort(-scores)
            pred_labels = classes[score_sort]
            pred_boxes  = boxes[score_sort]
            scores = scores[score_sort]

            if not (len(boxes) == len(scores) == len(classes)):
                print("!!!!!!!!!!")

            if not (len(boxes) == len(pred_boxes)):
                print("??????")

            num_preds += len(scores)

            if i % 100 == 0:
                print(i, "{}".format(time.time() - start))
                start = time.time()

            new_pred_boxes = [None for i in range(len(pred_boxes))]

            for ind in range(len(pred_boxes)):
                new_pred_boxes[ind] = np.append(pred_boxes[ind], scores[ind])

            pred_boxes = np.array(new_pred_boxes).copy()

            #print("all preds cnt", len(scores))

            d1 = {}
            for label in range(len(LABELS)):
                d1[label] = pred_boxes[pred_labels == label+1]

            cn_ = 0
            for label in range(len(LABELS)):
                all_detections[i][label] = d1[label].copy()
                cn_ += len(all_detections[i][label])
            #print("have", cn_)

            #print(all_detections[i])

            annotations = [[sign['xmin'], sign['ymin'], sign['xmax'], sign['ymax']] for sign in image_dict['signs']]

            d2 = {}

            for label in range(len(LABELS)):
                d2[label] = []
                for ind,sign in enumerate(image_dict['signs']):
                    if LABELS.index(sign['name']) == label:
                        d2[label].append(annotations[ind])

            for label in range(len(LABELS)):
                all_annotations[i][label] = d2[label].copy()
        
print("all predictions cnt:", num_preds)
print(12)
cnt = 0
for i in range(len(images)):
    for j in range(len(LABELS)):
        cnt += len(all_detections[i][j])
cnt
with open("all_detections_swedish_with_class", "w") as f:
    f.write(str(all_detections))
    
with open("all_annotations_swedish_with_class", "w") as f:
    f.write(str(all_annotations))
def compute_overlap(a, b):
    a = np.array(a)
    b = np.array(b)
    area = (b[:, 2] - b[:, 0]) * (b[:, 3] - b[:, 1])

    iw = np.minimum(np.expand_dims(a[:, 2], axis=1), b[:, 2]) - np.maximum(np.expand_dims(a[:, 0], 1), b[:, 0])
    ih = np.minimum(np.expand_dims(a[:, 3], axis=1), b[:, 3]) - np.maximum(np.expand_dims(a[:, 1], 1), b[:, 1])
        
    iw = np.maximum(iw, 0)
    ih = np.maximum(ih, 0)

    ua = np.expand_dims((a[:, 2] - a[:, 0]) * (a[:, 3] - a[:, 1]), axis=1) + area - iw * ih

    ua = np.maximum(ua, np.finfo(float).eps)
    
    
    intersection = iw * ih
    return intersection / ua  
    
def compute_ap(recall, precision):
    mrec = np.concatenate(([0.], recall, [1.]))
    mpre = np.concatenate(([0.], precision, [0.]))

    for i in range(mpre.size - 1, 0, -1):
        mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])

    i = np.where(mrec[1:] != mrec[:-1])[0]
    ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])
    return ap
print("start computing precisions")
        
average_precisions = {}
    
iou_threshold = 0.2
    
for label in range(len(LABELS)):
    
    false_positives = np.zeros((0,))
    true_positives  = np.zeros((0,))
    scores          = np.zeros((0,))
    num_annotations = 0
    num_preds = 0

    for i, image_dict in enumerate(images):
        detections           = all_detections[i][label]
        annotations          = all_annotations[i][label]
        num_annotations     += len(annotations)
        detected_annotations = []
        num_preds += len(detections)
        
        #print(detections)
        
        for d in detections:
            scores = np.append(scores, d[4])

            if len(annotations) == 0:
                print("false prediction")
                false_positives = np.append(false_positives, 1)
                true_positives  = np.append(true_positives, 0)
                continue

            overlaps            = compute_overlap(np.expand_dims([d[1], d[0], d[3], d[2]], axis=0), annotations)
            #print(overlaps)
            assigned_annotation = np.argmax(overlaps, axis=1)
            max_overlap         = overlaps[0, assigned_annotation]
            if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:
                print("accepted prediction")
                false_positives = np.append(false_positives, 0)
                true_positives  = np.append(true_positives, 1)
                detected_annotations.append(assigned_annotation)
            else:
                print("rejected prediction")
                false_positives = np.append(false_positives, 1)
                true_positives  = np.append(true_positives, 0)
                
    #print("predictions cnt for", label, ":", num_preds)
    
    # no annotations -> AP for this class is 0 (is this correct?)
    if num_annotations == 0:
        average_precisions[label] = 0
        continue
    
    # sort by score
    indices         = np.argsort(-scores)
    false_positives = false_positives[indices]
    true_positives  = true_positives[indices]
    
    # compute false positives and true positives
    false_positives = np.cumsum(false_positives)
    true_positives  = np.cumsum(true_positives)

    # compute recall and precision
    recall    = true_positives / float(num_annotations)
    precision = true_positives / np.maximum(true_positives + false_positives, np.finfo(np.float64).eps)

    # compute average precision
    average_precision  = compute_ap(recall, precision)  
    average_precisions[label] = average_precision
average_precisions<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1219572166215884, 'fraction_numerical': 0.013523481681829359, 'mean_word_length': 2.6732896816437117, 'pattern_counts': {'":': 1, '<': 28, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '180819', 'license': 'Apache-2.0', 'n_tokens_mistral': 5436, 'n_tokens_neox': 5169, 'n_words': 989, 'path': 'research/object_detection/diploma.ipynb', 'repo_name': 'tveritinova/models', 'size': '16268'}