<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Setup2&nbsp;&nbsp;Choosing a Person3&nbsp;&nbsp;Searching the Candidate Faces3.1&nbsp;&nbsp;Labeling Random Samples3.2&nbsp;&nbsp;Selecting Most Likely Faces3.3&nbsp;&nbsp;Discarding a Fraction of the Candidates4&nbsp;&nbsp;Inspecting the Results5&nbsp;&nbsp;Save Labels6&nbsp;&nbsp;Reset the Notebook  SetupBefore we begin, we need to load some dependencies and define some utility functions. Run all initialization cells before proceeding.<jupyter_code>%matplotlib inline

print('Loading libraries... Please wait.')

from IPython.display import display, clear_output
from IPython.core.pylabtools import figsize
figsize(12, 5)
import ipywidgets as widgets
import itertools
import io
import os
import sys
import pickle
import PIL.Image
import time
import traceback
import random
import math
import numpy as np
np.warnings.filterwarnings('ignore')
import matplotlib.pyplot as plt
import urllib.request as request
from datetime import date
from collections import namedtuple, Counter
from subprocess import check_call

from esper.prelude import *
from esper.widget import *
from esper.plot_util import tile_images
from esper.major_canonical_shows import MAJOR_CANONICAL_SHOWS
from esper import embed_google_images

import esper.face_embeddings as face_embeddings

LABELER_NAME_PREFIX = 'face-identity-uncommon:'

WIDGET_STYLE_ARGS = {'description_width': 'initial'}

ReferenceFaces = namedtuple(
    'ReferenceFaces', ['name', 'ids', 'embs', 'imgs'])

def show_reference_imgs(refs):
    tiled_imgs = tile_images(
        [cv2.resize(x, (100, 100)) for x in refs.imgs], 
        cols=10, blank_value=255)
    print('Your reference images for {}.'.format(refs.name))
    plt.figure()
    imshow(tiled_imgs)
    plt.tight_layout()
    plt.show()

def flatten(l):
    return [item for sublist in l for item in sublist]

def split_list(l, idx):
    return l[:idx], l[idx:]

def query_faces(ids):
    faces = Face.objects.filter(id__in=ids)
    return faces.values(
        'id', 'bbox_y1', 'bbox_y2', 'bbox_x1', 'bbox_x2',
        'frame__number', 'frame__video__id', 'frame__video__fps',
        'shot__min_frame', 'shot__max_frame')

def query_sample(qs, n):
    return qs.order_by('?')[:n]

def query_faces_result(faces, expand_bbox=0.05):
    """Replaces qs_to_result"""
    result = []
    for face in faces:
        if (face.get('shot__min_frame') is not None and 
                face.get('shot__max_frame') is not None):
            min_frame = int(
                (face['shot__min_frame'] + 
                 face['shot__max_frame']) / 2)
        else:
            min_frame = face['frame__number']
        face_result = {
            'type': 'flat', 'label': '', 
            'elements': [{
                'objects': [{
                    'id': face['id'],
                    'background': False,
                    'type': 'bbox',
                    'bbox_y1': max(face['bbox_y1'] - expand_bbox, 0),
                    'bbox_y2': min(face['bbox_y2'] + expand_bbox, 1),
                    'bbox_x1': max(face['bbox_x1'] - expand_bbox, 0),
                    'bbox_x2': min(face['bbox_x2'] + expand_bbox, 1),
                }], 
                'min_frame': min_frame,
                'video': face['frame__video__id']
            }]
        }
        result.append(face_result)
    return {'type': 'Face', 'count': 0, 'result': result}

def load_face_img(face):
    return crop(load_frame(face.frame.video, face.frame.number, []), face)

def sort_ids_by_distance(ids, embs):
    dists = face_embeddings.dist(ids, targets=embs)
    return [i for _, i in sorted(zip(dists, ids))]

def continue_yn_prompt(msg):
    l = input('{} Continue? (y/N): '.format(msg))
    if l.strip().lower() != 'y':
        raise ValueError('User entered No. This is not an error.') 

def load_and_select_faces_from_images(name, img_dir):
    
    def crop_img(img, bbox):
        height, width, _ = img.shape
        y1 = int(bbox.y1 * height)
        y2 = int(bbox.y2 * height)
        x1 = int(bbox.x1 * width)
        x2 = int(bbox.x2 * width)
        return img[y1:y2, x1:x2, :]
    
    face_bboxes = embed_google_images.detect_faces(img_dir)
    
    cand_imgs = []
    for img_path, bbox in face_bboxes:
        img = cv2.imread(img_path)
        img_crop = crop_img(img, bbox)
        assert img_crop.size > 0, \
            'Bad crop dimensions: {} from {}'.format(
            img_crop.shape, bbox)
        img_crop = cv2.resize(img_crop, (100, 100))
        cand_imgs.append(img_crop)
    
    def img_to_widget(img):
        height, width, _ = img.shape
        f = io.BytesIO()
        PIL.Image.fromarray(img).save(f, 'png')
        return widgets.Image(value=f.getvalue(), height=height,
                             width=width)
    
    def get_img_checkbox():
        img_checkbox = widgets.ToggleButton(
            layout=widgets.Layout(width='auto'),
            value=False,
            description='',
            disabled=False,
            button_style='',
            icon=''
        )
        def on_toggle(b):
            if img_checkbox.value:
                img_checkbox.button_style = 'danger'
                img_checkbox.icon = 'check'
            else:
                img_checkbox.button_style = ''
                img_checkbox.icon = ''
        img_checkbox.observe(on_toggle, names='value')
        return img_checkbox
    
    print('Select reference images below: (default=selected)')
    checkboxes = []
    vboxes = []
    for img_crop in cand_imgs:
        img_widget = img_to_widget(
            cv2.cvtColor(img_crop, cv2.COLOR_BGR2RGB))
        img_checkbox = get_img_checkbox()
        checkboxes.append(img_checkbox)
        vboxes.append(widgets.VBox([img_widget, img_checkbox]))
    
    images_per_row = 8
    for i in range(0, len(vboxes), images_per_row):
        display(widgets.HBox(vboxes[i:i + images_per_row]))
        
    submit_button = widgets.Button(
        layout=widgets.Layout(width='auto'),
        style=WIDGET_STYLE_ARGS,
        description='Confirm selections',
        disabled=False,
        button_style='danger'
    )
    def on_submit(b):
        imgs = [
            x for i, x in enumerate(cand_imgs) 
            if checkboxes[i].value]
        clear_output()
        print('Selected {} faces. Ignored {}.'.format(
              len(imgs), len(cand_imgs) - len(imgs)))
        embs = embed_google_images.embed_images(imgs)
        assert len(imgs) == len(embs)
        global FACE_REFERENCES
        FACE_REFERENCES = ReferenceFaces(
            name=name, ids=set(), embs=embs, imgs=imgs)
        show_reference_imgs(FACE_REFERENCES)
    submit_button.on_click(on_submit)
    
    cancel_button = widgets.Button(
        layout=widgets.Layout(width='auto'),
        style=WIDGET_STYLE_ARGS,
        description='Abort selection',
        disabled=False,
        button_style='warning'
    )
    def on_cancel(b):
        clear_output()
        print('Canceled selection. No references images were added.')
    cancel_button.on_click(on_cancel)
    
    display(widgets.HBox([widgets.Label('Controls:'), 
            submit_button, cancel_button]))
    return cand_imgs

def get_google_images(name, n=10, **kwargs):
    img_dir = embed_google_images.fetch_images(name, n=n, force='query_extras' in kwargs, 
                                               **kwargs)
    load_and_select_faces_from_images(name, img_dir)

LabelingState = namedtuple('LabelingState', ['references', 'faces', 'candidate_ids', 'selected_ids'])

def get_new_labeling_state(face_references, max_threshold=1.):
    start_time = time.time()
    results = face_embeddings.knn(
        targets=FACE_REFERENCES.embs, 
        k=max_faces(), 
        max_threshold=max_threshold
    )
    knn_time = time.time()
    print('Comuputed k-NN: {:0.4f}s'.format(knn_time - start_time))
    candidate_face_ids = [i for i, _ in results]
    candidate_faces = list(query_faces(candidate_face_ids))
    print('Database query: {:0.4f}s'.format(time.time() - start_time))
    print('Retreived {} candidate faces'.format(len(candidate_faces)))
    return LabelingState(
        references=face_references,
        faces={f['id']: f for f in candidate_faces},
        candidate_ids={f['id'] for f in candidate_faces},
        selected_ids=set())

def print_status(state):
    print('\nThere are now {} labeled "{}"s and {} unlabeled faces remaining.'.format(
        len(state.selected_ids), state.references.name, 
        len(state.candidate_ids)))

def save_labels(state):
    name = state.references.name
    print('Attempting to save {} labels for "{}" to the database.'.format(
        len(state.selected_ids), name))
    labeler_name = '{}{}'.format(LABELER_NAME_PREFIX, name.lower())
    labeler, created = Labeler.objects.get_or_create(name=labeler_name)
    print('Created labeler:' if created else 'Found existing labeler:', labeler.name)
    identity, created = Identity.objects.get_or_create(name=name.lower())
    print('Created identity:' if created else 'Found existing identity:', identity.name)

    previously_labeled_ids = {
        f['face__id'] for f in 
        FaceIdentity.objects.filter(labeler=labeler).values('face__id')
    }
    if len(previously_labeled_ids) > 0:
        print('Note: there are already {} labels for "{}" in the database'.format(
            len(previously_labeled_ids), name))

    face_idents = []
    for i in state.selected_ids:
        if i in previously_labeled_ids:
            continue
        face_idents.append(FaceIdentity(
            face_id=i, identity=identity, labeler=labeler, probability=1))
    FaceIdentity.objects.bulk_create(face_idents)
    print('Saved {} new labels.'.format(len(face_idents)))
    
def reset_notebook():
    global FACE_REFERENCES, STATE
    try: del FACE_REFERENECES
    except NameError: pass
    try: del STATE
    except NameError: pass
    print('Reset notebook. Specify a new person to restart labeling.')
    
DEFAULT_FACES_PER_PAGE = 50
_faces_per_page_slider = widgets.IntSlider(
    value=DEFAULT_FACES_PER_PAGE,
    style=WIDGET_STYLE_ARGS,
    min=25,
    max=100,
    step=1,
    description='Faces per widget page:',
    disabled=False,
    continuous_update=False,
    orientation='horizontal',
    readout=True,
    readout_format='d'
)
def faces_per_page():
    try:
        return _faces_per_page_slider.value
    except:
        return DEFAULT_FACES_PER_PAGE

DEFAULT_MAX_FACES = 100000
_max_faces_slider = widgets.IntSlider(
    value=DEFAULT_MAX_FACES,
    style=WIDGET_STYLE_ARGS,
    min=10000,
    max=200000,
    step=1000,
    description='Max candidates:',
    disabled=False,
    continuous_update=False,
    orientation='horizontal',
    readout=True,
    readout_format='d'
)
def max_faces():
    try:
        return _max_faces_slider.value
    except:
        return DEFAULT_MAX_FACES

display(_faces_per_page_slider)
display(_max_faces_slider)

print('Done!')<jupyter_output>Loading libraries... Please wait.<jupyter_text>Choosing a PersonFirst, we will find images of a person using Google Image Search. These images are necessary to communicate to the model the appearance of the person. Enter a name below and any extra terms needed to find the person on Google.<jupyter_code>name_text = widgets.Text(
    value='',
    placeholder='e.g., <NAME>',
    style=WIDGET_STYLE_ARGS,
    description='Name:',
    disabled=False
)
search_terms_text = widgets.Text(
    value='',
    placeholder='e.g., CNN anchor',
    style=WIDGET_STYLE_ARGS,
    description='Extra terms (optional):',
    disabled=False
)
submit_button = widgets.Button(
    description='Search',
    disabled=False,
    button_style='danger'
)
google_search_output = widgets.Output()
def on_submit(b):
    name = name_text.value.strip()
    if len(name) == 0:
        print('Name cannot be empty. Try again.', file=sys.stderr)
        return
    search_terms = search_terms_text.value.strip()
    with google_search_output:
        clear_output()
        print('Seaching for "{}"'.format(name))
        if search_terms != '':
            print('Extra terms:', search_terms)
        get_google_images(name, query_extras=search_terms)
submit_button.on_click(on_submit)
display(name_text)
display(search_terms_text)
display(submit_button)
display(google_search_output)<jupyter_output><empty_output><jupyter_text>Once you are satisfied with the reference images, run the cell below to retrieve a conservative set of candidate faces and initialize the labeling state.<jupyter_code>STATE = get_new_labeling_state(FACE_REFERENCES)<jupyter_output><empty_output><jupyter_text>Searching the Candidate FacesGiven that we now have a set of candidate faces that may be the person we are after, we will now pickout selections and filter down the candidate set until we are satisfied that we have found most of the instances of the target person in the dataset.To refine the selections, there are three operations currently supported. These can be done in any order by running the code below.- Labeling randomly sample candidates- Labeling the most likely candidates- Discarding the least likely candidates<jupyter_code>print('Initializing libraries. Please wait...')

def get_random_samples(state, k=None):
    if k is None:
        k = faces_per_page()
    
    start_time = time.time()
    samples = random.sample(state.candidate_ids, k)
    
    submit_button = widgets.Button(
        layout=widgets.Layout(width='auto'),
        style=WIDGET_STYLE_ARGS,
        description='Confirm selections',
        disabled=False,
        button_style='danger',
    )

    samples_ord = sort_ids_by_distance(samples, state.references.embs)
    samples_faces = [state.faces[i] for i in samples_ord]
    
    selection_widget = esper_widget(
        query_faces_result(samples_faces), results_per_page=faces_per_page(),
        crop_bboxes=True, jupyter_keybindings=True, disable_playback=True,
        show_inline_metadata=False,
        disable_captions=True
    )

    cand_count = len(state.candidate_ids)
    def on_submit(b):
        # Read from the widget, update selections, and commit result
        selected_idxs = set(selection_widget.selected)
        selection_widget.close()
        clear_output() 
        
        # Add to positive set
        pos_samples = {
            f['id'] for _, f in filter(
                lambda x: x[0] in selected_idxs,
                enumerate(samples_faces))
        }

        # Add to negative set
        neg_samples = {
            f['id'] for _, f in filter(
                lambda x: x[0] not in selected_idxs,
                enumerate(samples_faces))
        }
        
        print('You selected {} and ignored {} faces.'.format(
              len(pos_samples), len(neg_samples)))
        
        # Update state
        for i, e in face_embeddings.get(list(pos_samples)):
            if i not in state.references.ids:
                state.references.embs.append(e)
        state.references.ids.update(pos_samples)
        state.selected_ids.update(pos_samples)
        state.candidate_ids.difference_update(pos_samples)
        state.candidate_ids.difference_update(neg_samples)
        
        # Compute expected number of true positives
        p_hat = len(pos_samples) / (len(pos_samples) + len(neg_samples))
        p_hat_smooth = (len(pos_samples) + 1) / (len(pos_samples) + len(neg_samples) + 2)
        print('\nEstimated number of "{}"s remaining in the dataset: {} +/- {}'.format(
            state.references.name,
            int(p_hat * cand_count),
            int(cand_count * 1.96 * math.sqrt(
                p_hat_smooth * (1 - p_hat_smooth) / (len(pos_samples) + len(neg_samples) + 2)))
        ))
        print_status(state)
    submit_button.on_click(on_submit)
    
    display(widgets.HBox([widgets.Label('Controls:'), submit_button]))
    print('You MUST select all faces that are "{}". (Sorted from most to least similar.)'.format(state.references.name))
    display(selection_widget)

def do_discard(state):
    fraction = float(input('Enter a proportion to discard (0, 1): '))
    assert fraction > 0 and fraction < 1, 'Invalid discard fraction'
    count = len(state.candidate_ids)
    k = math.ceil((1. - fraction) * count)
    
    candidate_ids_ord = sort_ids_by_distance(
        list(state.candidate_ids), state.references.embs
    )[:k]
    
    # Update state
    state.candidate_ids.intersection_update(candidate_ids_ord)
    
    print('Discarded {} candidates out of {}.'.format(
        count - len(candidate_ids_ord), count))
    print_status(state)
    
def do_nn_search(state, k=10000):
    candidate_ids_ord = sort_ids_by_distance(
        list(state.candidate_ids), state.references.embs
    )[:k]
    candidate_faces = [state.faces[i] for i in candidate_ids_ord]
    
    dismiss_button = widgets.Button(
        layout=widgets.Layout(width='auto'),
        style=WIDGET_STYLE_ARGS,
        description='Dismiss widget',
        disabled=False,
        button_style='',
    )
    def on_dismiss(b):
        clear_output()
        print('Dismissed widget. No faces have been selected.')
    dismiss_button.on_click(on_dismiss)

    selection_widget = esper_widget(
        query_faces_result(candidate_faces), results_per_page=faces_per_page(),
        crop_bboxes=True, jupyter_keybindings=True, disable_playback=True,
        show_inline_metadata=False,
        disable_captions=True
    )
    
    submit_button = widgets.Button(
        layout=widgets.Layout(width='auto'),
        style=WIDGET_STYLE_ARGS,
        description='Confirm selections',
        disabled=False,
        button_style='danger',
    )
    def on_submit(b):
        selected_idxs = set(selection_widget.selected)
        ignored_idxs = set(selection_widget.ignored)
        if len(selected_idxs) == 0:
            selected_idxs = {i for i in range(len(candidate_faces)) 
                             if i not in ignored_idxs}
        clear_output()
        
        pos_samples = {candidate_faces[i]['id'] for i in selected_idxs}
        neg_samples = {candidate_faces[i]['id'] for i in ignored_idxs}
        
        # Update state
        state.selected_ids.update(pos_samples)
        state.candidate_ids.difference_update(pos_samples)
        state.candidate_ids.difference_update(neg_samples)
        
        print('Added {} examples and removed {} non-examples of "{}".'.format(
            len(pos_samples), len(neg_samples), state.references.name))
        print_status(state)
    submit_button.on_click(on_submit)
    
    display(widgets.HBox([widgets.Label('Controls:'), submit_button, dismiss_button]))
    print('You should select faces that are "{}". (Sorted from most to least similar.)'.format(
        state.references.name))
    display(selection_widget)
    
def show_selections(state):
    dismiss_button = widgets.Button(
        layout=widgets.Layout(width='auto'),
        style=WIDGET_STYLE_ARGS,
        description='Dismiss widget',
        disabled=False,
        button_style='',
    )
    def on_dismiss(b):
        clear_output()
        print('Dismissed widget. Rerun the cell to get it back.')
    dismiss_button.on_click(on_dismiss)
    
    selected_ids = list(state.selected_ids)
    mean_selected_emb = face_embeddings.mean(selected_ids)
    selected_ids = sort_ids_by_distance(selected_ids, [mean_selected_emb])[::-1]
    selected_faces = [state.faces[i] for i in selected_ids]

    selection_widget = esper_widget(
        query_faces_result(selected_faces), disable_playback=True,
        crop_bboxes=True, jupyter_keybindings=True, show_inline_metadata=False, disable_captions=True,
        results_per_page=faces_per_page())

    submit_button = widgets.Button(
        layout=widgets.Layout(width='auto'),
        style=WIDGET_STYLE_ARGS,
        description='Update selections',
        disabled=False,
        button_style='danger',
    )
    def on_submit(b):
        selected_idxs = set(selection_widget.selected)
        ignored_idxs = set(selection_widget.ignored)

        selected_ids = {
            f['id'] for i, f in enumerate(selected_faces) if i in selected_idxs
        }
        ignored_ids = {
            f['id'] for i, f in enumerate(selected_faces) if i in ignored_idxs
        }
        
        # Update state
        for i, e in face_embeddings.get(list(selected_ids)):
            if i not in state.references.ids:
                state.references.embs.append(e)
        state.references.ids.update(selected_ids)
        state.references.ids.difference_update(ignored_ids)
        state.selected_ids.difference_update(ignored_ids)

        clear_output()
        print(('Removed {} faces selected set. Added {} faces to the NN examples.').format(
            len(ignored_ids), len(selected_ids)))
    submit_button.on_click(on_submit)
    
    display(widgets.HBox([
        widgets.Label('Controls:'), submit_button, dismiss_button]))
    display(selection_widget)

print('Done!')<jupyter_output><empty_output><jupyter_text>Labeling Random SamplesLabeling faces sampled randomly from the candidate set is a good way to obtain diversity in selected faces and also estimate the number of faces remaining in the dataset.`get_random_samples()` randomly samples faces from the candidate set. You MUST select the faces that are of the target person.Instructions:- Use "[" to select faces that are the target person (yellow highlight).- Hit confirm when done.Once you have confirmed, the selections will be saved and the number of remaining faces will be shown (estimated to 95% confidence).<jupyter_code>get_random_samples(STATE)<jupyter_output><empty_output><jupyter_text>Selecting Most Likely Faces`do_nn_search()` will retreive faces in the candidate set and display most likely instances of the person first. Your task is to select faces that are the target person.Instructions:- Use "[" to select individiual faces (yellow highlight). "Shift + [" to highlight all unhighlighted faces on a page.- Use "Shift + ?" to select all unhighlighted faces up to the current one.- Use "]" to discard individual faces (red highlight). "Shift + ]" to discard all unhighlighted faces on a page.- Hit confirm when done.- If no faces are selected, then all unhighlighted faces will be selected.All faces highlighted in yellow will be saved, and removed from the candidate set. All faces highlighted in red will be discarded.<jupyter_code>do_nn_search(STATE)<jupyter_output><empty_output><jupyter_text>Discarding a Fraction of the CandidatesIf the target person is expected to occur 10K times and we have 100K candidates, we may potentially want to discard the least likely candidates. `do_discard()` removes the least likely fraction of the candidates.<jupyter_code>do_discard(STATE)<jupyter_output><empty_output><jupyter_text>Inspecting the Results`show_selections()` displays all of your current selections, in order from most likely to be a mistake to least likely.Instructions:- Use "[" to select individiual faces (yellow highlight). These will be added to the examples for whick NN search uses.- Use "]" to discard individual faces (red highlight). These will be unselected and discarded.- Hit confirm when done.<jupyter_code>show_selections(STATE)<jupyter_output><empty_output><jupyter_text>Save LabelsThis will save your selections to the database.<jupyter_code>save_labels(STATE)<jupyter_output><empty_output><jupyter_text>Reset the Notebook<jupyter_code>reset_notebook()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10234142188165177, 'fraction_numerical': 0.0060025542784163475, 'mean_word_length': 3.1672875643072556, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '285572', 'license': 'Apache-2.0', 'n_tokens_mistral': 7372, 'n_tokens_neox': 7130, 'n_words': 1697, 'path': 'app/notebooks/identity_labeling_uncommon.ipynb', 'repo_name': 'scanner-research/esper-tv', 'size': '23492'}