<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;<NAME>2&nbsp;&nbsp;<NAME>3&nbsp;&nbsp;<NAME>4&nbsp;&nbsp;<NAME>5&nbsp;&nbsp;<NAME>6&nbsp;&nbsp;<NAME>7&nbsp;&nbsp;<NAME>8&nbsp;&nbsp;<NAME>9&nbsp;&nbsp;<NAME>10&nbsp;&nbsp;<NAME>11&nbsp;&nbsp;<NAME>12&nbsp;&nbsp;<NAME> in AC360<jupyter_code>from query.datasets.prelude import *
from query.datasets.ingest import ingest_pose
import pyspark.sql.functions as func
def face_img(face):
    return crop(load_frame(face.person.frame.video, face.person.frame.number, []), face)

def faces_to_tiled_img(faces):
    def tile(imgs, rows=None, cols=None):
        # If neither rows/cols is specified, make a square
        if rows is None and cols is None:
            rows = int(math.sqrt(len(imgs)))

        if rows is None:
            rows = int((len(imgs) + cols - 1) / cols)
        else:
            cols = int((len(imgs) + rows - 1) / rows)

        # Pad missing frames with black
        diff = rows * cols - len(imgs)
        if diff != 0:
            imgs.extend([np.zeros(imgs[0].shape, dtype=imgs[0].dtype) for _ in range(diff)])

        return np.vstack([np.hstack(imgs[i * cols:(i + 1) * cols]) for i in range(rows)])
    
    face_imgs = par_for(face_img, faces)
    im = tile([cv2.resize(img, (200, 200)) for img in face_imgs], cols=12)
    return im
def face_search_by_ids(target_face_ids, not_target_face_ids, 
                       increment=0.05, min_thresh=0., max_thresh=1.,
                       max_results_per_group=72, exclude_labeled=False,
                       show_name=None):
    face_qs = UnlabeledFace.objects if exclude_labeled else Face.objects

    face_sims = face_knn(ids=target_face_ids, min_threshold=min_thresh, max_threshold=max_thresh,
                         not_ids=not_target_face_ids)
    
    if show_name is not None:
        face_set = set(x['id'] for x in Face.objects.filter(shot__video__show__name=show_name).values('id'))
    else:
        face_set = None

    results_by_bucket = {}
    for t in frange(min_thresh, max_thresh, increment):
        face_ids = [x for x, _ in filter(lambda z: z[1] >= t and z[1] < t + increment, face_sims)]
        
        if face_set is not None:
            face_ids = [x for x in filter(lambda x: x in face_set, face_ids)]
            
        if len(face_ids) != 0:
            faces = face_qs.filter(
                id__in=random.sample(face_ids, k=min(len(face_ids), max_results_per_group))
            ).select_related('person__frame__video', 'person__frame')
            results_by_bucket[(t, t + increment, len(face_ids))] = (face_ids, faces)

    if len(results_by_bucket) == 0:
        raise Exception('No results to show')
        
    return results_by_bucket
import concurrent.futures

def get_num_from_user(msg, a, b):
    while True:
        try:
            v = input(msg)
            if v == 'q':
                return b, False
            if v == 'qq':
                return b, True
            if v == 'a':
                return a, False
            if v == 'aa':
                return a, True
            v = int(v)
            if v >= a and v <= b:
                return v, False
        except:
            pass

def get_precision_by_bucket(results_by_bucket):
    precision_by_bucket = {}
    tiled_im_futures_by_bucket = {}
    
    with concurrent.futures.ThreadPoolExecutor(2) as executor:
        sorted_keys = sorted(results_by_bucket.keys())
        lower_keys = sorted_keys[:int(len(sorted_keys) / 2)][::-1]
        upper_keys = sorted_keys[len(lower_keys):]
        
        for k in lower_keys:
            faces = results_by_bucket[k][1]
            tiled_im_futures_by_bucket[k] = executor.submit(faces_to_tiled_img, list(faces))
            
        for k in upper_keys:
            faces = results_by_bucket[k][1]
            tiled_im_futures_by_bucket[k] = executor.submit(faces_to_tiled_img, list(faces))
    
        print('\nDescending:')
        is_done = False
        for k in lower_keys:
            faces = results_by_bucket[k][1]
            faces_count = faces.count()
            if not is_done:
                print('in range=({:0.2f}, {:0.2f}), total={}'.format(k[0], k[1], k[2]))
                im = tiled_im_futures_by_bucket[k].result()
                imshow(im)
                plt.show()

                num_mistakes, done = get_num_from_user(
                    'How many mistakes (max={})? '.format(faces_count), 0, faces_count)
                if done:
                    is_done = True
            else:
                # No mistakes
                num_mistakes = 0
            precision_by_bucket[k] = (faces_count - num_mistakes) / faces_count
    
        print('\nAscending:')
        is_done = False
        for k in upper_keys:
            faces = results_by_bucket[k][1]
            faces_count = faces.count()
            if not is_done:
                print('in range=({:0.2f}, {:0.2f}), total={}'.format(k[0], k[1], k[2]))
                im = tiled_im_futures_by_bucket[k].result()
                imshow(im)
                plt.show()

                num_mistakes, done = get_num_from_user(
                    'How many mistakes (max={})? '.format(faces_count), 0, faces_count)
                if done:
                    is_done = True
            else:
                # All mistakes
                num_mistakes = faces_count
            precision_by_bucket[k] = (faces_count - num_mistakes) / faces_count

    return precision_by_bucket
def compute_exp_positives(results_by_bucket, precision_by_bucket, verbose=True):
    exp_total_positives = 0
    exp_positives_by_bucket = {}
    for k in sorted(results_by_bucket.keys()):
        min_t, max_t, count = k
        if verbose:
            print('in range=({:0.2f}, {:0.2f})'.format(min_t, max_t))
        exp_positives = precision_by_bucket[k] * count
        if verbose:
            print('\tprecision={:0.3f}, count={}, exp_positives={:0.2f}'.format(precision_by_bucket[k], count, exp_positives))
        exp_total_positives += exp_positives
        exp_positives_by_bucket[k] = exp_positives
    if verbose:
        print('Expected positives:', exp_total_positives)
    return exp_positives_by_bucket

def plot_precision_and_cdf(precision_by_bucket, exp_positives_by_bucket):
    x = []
    
    y_cdf = []
    acc = 0.
    for k in sorted(exp_positives_by_bucket.keys()):
        min_t, max_t, count = k
        x.append((max_t + min_t) / 2.)
        acc += exp_positives_by_bucket[k]
        y_cdf.append(acc)
    y_cdf = [z / acc for z in y_cdf]
    plt.plot(x, y_cdf, '-o', color='blue', label='cdf')
    
    y_prec = []
    for k in sorted(precision_by_bucket.keys()):
        y_prec.append(precision_by_bucket[k])
    plt.plot(x, y_prec, '-x', color='red', label='precision')
    
    plt.xlabel('Distance threshold')
    plt.legend()
    
    plt.show()
import _pickle as pickle
import os

RESULTS_DIR = '/app/data/face_eval'
if not os.path.exists(RESULTS_DIR):
    os.makedirs(RESULTS_DIR)

def save_results(name, results_by_bucket, precision_by_bucket):
    out_path = os.path.join(RESULTS_DIR, '{}.pkl'.format(name))
    assert not os.path.exists(out_path)
    with open(out_path, 'wb') as f:
        pickle.dump([results_by_bucket, precision_by_bucket], f)
        
def load_results(name):
    in_path = os.path.join(RESULTS_DIR, '{}.pkl'.format(name))
    with open(in_path, 'rb') as f:
        obj = pickle.load(f)
    return obj[0], obj[1]
# Wrap all parts up in a single command
def run_all(name, ids, not_ids):
    try:
        results_by_bucket, precision_by_bucket = load_results(name)
    except:
        results_by_bucket = face_search_by_ids(ids, not_ids)
        precision_by_bucket = get_precision_by_bucket(results_by_bucket)
    exp_pos_by_bucket = compute_exp_positives(
        results_by_bucket, precision_by_bucket)
    plot_precision_and_cdf(precision_by_bucket, exp_pos_by_bucket)
    save_results(name, results_by_bucket, precision_by_bucket)<jupyter_output><empty_output><jupyter_text>Wolf Blitzer<jupyter_code># Wolf Blitzer
wolf_blitzer = [975965, 5254043, 844004, 105093, 3801699, 4440669, 265071]
not_wolf_blitzer =  [
    1039037, 3132700, 3584906, 2057919, 3642645, 249473, 129685, 2569834, 5366608,
    4831099, 2172821, 1981350, 1095709, 4427683, 1762835]
wolf_blitzer_results_by_bucket = face_search_by_ids(wolf_blitzer, not_wolf_blitzer)
wolf_blitzer_precision_by_bucket = get_precision_by_bucket(wolf_blitzer_results_by_bucket)
wolf_blitzer_exp_pos_by_bucket = compute_exp_positives(
    wolf_blitzer_results_by_bucket, wolf_blitzer_precision_by_bucket)
plot_precision_and_cdf(wolf_blitzer_precision_by_bucket, wolf_blitzer_exp_pos_by_bucket)
save_results('Wolf Blitzer', wolf_blitzer_results_by_bucket, wolf_blitzer_precision_by_bucket)
wolf_blitzer_results_by_bucket, wolf_blitzer_precision_by_bucket = load_results('Wolf Blitzer')<jupyter_output><empty_output><jupyter_text>Melania Trump<jupyter_code># Melania Trump
melania_trump = [
    2869846, 3851770, 3567361, 401073, 3943919, 5245641, 198592, 5460319, 5056617,
    1663045, 3794909, 1916340, 1373079, 2698088, 414847, 4608072]
not_melania_trump = []
melania_trump_results_by_bucket = face_search_by_ids(melania_trump, not_melania_trump)
melania_trump_precision_by_bucket = get_precision_by_bucket(melania_trump_results_by_bucket)
melania_trump_exp_pos_by_bucket = compute_exp_positives(
    melania_trump_results_by_bucket, melania_trump_precision_by_bucket)
plot_precision_and_cdf(melania_trump_precision_by_bucket, melania_trump_exp_pos_by_bucket)
melania_trump_results_by_bucket, melania_trump_precision_by_bucket = load_results('Melania Trump')<jupyter_output><empty_output><jupyter_text><NAME><jupyter_code># <NAME>
bernie_sanders = [
    644710, 4686364, 2678025, 62032, 13248, 4846879, 4804861, 561270, 2651257, 
    2083010, 2117202, 1848221, 2495606, 4465870, 3801638, 865102, 3861979, 4146727, 
    3358820, 2087225, 1032403, 1137346, 2220864, 5384396, 3885087, 5107580, 2856632,
    335131, 4371949, 533850, 5384760, 3335516]
not_bernie_sanders = [
    2656438, 1410140, 4568590, 2646929, 1521533, 1212395, 178315, 1755096, 3476158,
    3310952, 1168204, 3062342, 1010748, 1275607, 2190958, 2779945, 415610, 1744917, 
    5210138, 3288162, 5137166, 4169061, 3774070, 2595170, 382055, 2365443, 712023, 
    5214225, 178251, 1039121, 5336597, 525714, 4522167, 3613622, 5161408, 2091095, 
    741985, 521, 2589969, 5120596, 284825, 3361576, 1684384, 4437468, 5214225, 
    178251]
bernie_sanders_results_by_bucket = face_search_by_ids(bernie_sanders, not_bernie_sanders)
bernie_sanders_precision_by_bucket = get_precision_by_bucket(bernie_sanders_results_by_bucket)
bernie_sanders_exp_pos_by_bucket = compute_exp_positives(
    bernie_sanders_results_by_bucket, bernie_sanders_precision_by_bucket)
plot_precision_and_cdf(bernie_sanders_precision_by_bucket, bernie_sanders_exp_pos_by_bucket)
bernie_sanders_results_by_bucket, bernie_sanders_precision_by_bucket = load_results('<NAME>')<jupyter_output><empty_output><jupyter_text><NAME><jupyter_code>mark_zuckerberg = [
    4373010, 43236, 3381795, 831835, 2248811, 141947, 1211853, 2483825, 3775530, 392015, 
    4285710, 4546500, 4546503]
not_mark_zuckerberg = [2316891, 1939793, 4874772, 1432980, 1361759, 3177171]
mark_zuckerberg_results_by_bucket = face_search_by_ids(mark_zuckerberg, not_mark_zuckerberg)
mark_zuckerberg_precision_by_bucket = get_precision_by_bucket(mark_zuckerberg_results_by_bucket)
mark_zuckerberg_exp_pos_by_bucket = compute_exp_positives(
    mark_zuckerberg_results_by_bucket, mark_zuckerberg_precision_by_bucket)
plot_precision_and_cdf(mark_zuckerberg_precision_by_bucket, mark_zuckerberg_exp_pos_by_bucket)
mark_zuckerberg_results_by_bucket, mark_zuckerberg_precision_by_bucket = load_results('<NAME>')<jupyter_output><empty_output><jupyter_text><NAME><jupyter_code>tim_kaine = [4250187,431813,1502784,960947,3330893,3179555,5376026,1975421,4294387,4048666,970862,791444,3151748,723687,3639949,3688139,1982437,3669192,3297978,3413150,4671858,3639932,4849192]
not_tim_kaine = [675167,1165340,4715362,3652401,2525138,1104418,205970,1492495,1534610,2673289,5130988]
tim_kaine_results_by_bucket = face_search_by_ids(tim_kaine, not_tim_kaine)
tim_kaine_precision_by_bucket = get_precision_by_bucket(tim_kaine_results_by_bucket)
tim_kaine_exp_pos_by_bucket = compute_exp_positives(
    tim_kaine_results_by_bucket, tim_kaine_precision_by_bucket)
plot_precision_and_cdf(tim_kaine_precision_by_bucket, tim_kaine_exp_pos_by_bucket)
save_results('<NAME>', tim_kaine_results_by_bucket, tim_kaine_precision_by_bucket)
tim_kaine_results_by_bucket, tim_kaine_precision_by_bucket = load_results('<NAME>')<jupyter_output><empty_output><jupyter_text><NAME><jupyter_code>robert_mueller = [5388412, 502130, 3679351, 640339, 3420367, 3792778, 1101977, 2394643]
not_robert_mueller = [1095302, 4876120, 5184388, 153296, 4030256, 2430521, 2206059, 5051195]
robert_mueller_results_by_bucket = face_search_by_ids(robert_mueller, not_robert_mueller)
robert_mueller_precision_by_bucket = get_precision_by_bucket(robert_mueller_results_by_bucket)
robert_mueller_exp_pos_by_bucket = compute_exp_positives(
    robert_mueller_results_by_bucket, robert_mueller_precision_by_bucket)
plot_precision_and_cdf(robert_mueller_precision_by_bucket, robert_mueller_exp_pos_by_bucket)
save_results('<NAME>', robert_mueller_results_by_bucket, robert_mueller_precision_by_bucket)
robert_mueller_results_by_bucket, robert_mueller_precision_by_bucket = load_results('<NAME>')<jupyter_output><empty_output><jupyter_text><NAME><jupyter_code>rachael_maddow = [5411932, 5026969, 227347, 5397615, 2610932, 5143813, 4768306, 2986792, 1776363, 3660788, 2985833, 3614601, 738546]
not_rachael_maddow = [1110423, 3124197, 768092, 41168, 2658839, 1741497, 2604057, 2632174, 363196, 929804, 124162, 3702495, 41162]
run_all('<NAME>', rachael_maddow, not_rachael_maddow)<jupyter_output><empty_output><jupyter_text>Shinzo Abe<jupyter_code>shinzo_abe = [3871152, 634706, 3419912, 4041465, 4142150, 2463291, 1504197, 760858, 2029171, 4809661, 2856044, 248051]
not_shinzo_abe = [2119604, 3948631, 4332558, 2590500, 2398886, 1524931, 1923824, 99768, 3665314, 493627, 1490469, 3334990, 1121527, 147145, 518555, 2896594, 1174325]
run_all('Shinzo Abe', shinzo_abe, not_shinzo_abe)<jupyter_output><empty_output><jupyter_text>Poppy Harlow<jupyter_code>poppy_harlow = [1153, 73947, 2097382, 5224901, 1069781, 1156918, 1958703, 1283773, 690170]
not_poppy_harlow = [4907666, 494513, 1624647, 1267344, 4183896, 3114779, 4885324, 3553942, 2583807, 2979192, 3181136]
run_all('Poppy Harlow', poppy_harlow, not_poppy_harlow)<jupyter_output><empty_output><jupyter_text><NAME><jupyter_code>tucker_carlson = [5267247, 3225084, 1125109, 3391513, 2403468, 103730]
not_tucker_carlson = [4272014, 3251057, 2084769, 1556206, 634024, 4386334, 2090001, 1246434, 4859090, 1101704, 4956841]
run_all('<NAME>', tucker_carlson, not_tucker_carlson)<jupyter_output><empty_output><jupyter_text><NAME><jupyter_code>kim_jong_un = [5106259, 3238686, 507683, 1281110, 2580573, 2094955, 5219267, 662398, 1790791, 225639, 2077157]
not_kim_jong_un = [4807883, 3922406, 3599954, 5299738, 4187500, 2306802, 70677, 4675639, 4392802, 1572330, 3888002]
run_all('Kim Jong Un', kim_jong_un, not_kim_jong_un)<jupyter_output><empty_output><jupyter_text><NAME> in AC360<jupyter_code># <NAME>
anderson_cooper = [1135984,5437878,3342427,205360,4350698,2543743,3454099,893188,126536,1727260,1432464,1890665,5437853,3291560,2546041,3054775]
not_anderson_cooper = [598616, 2678741, 2519536, 1573692, 2930452, 2414707, 4247803, 38290, 2255930, 3185720, 5150502]
anderson_cooper_results_by_bucket = face_search_by_ids(anderson_cooper, not_anderson_cooper, show_name='<NAME> 360')
anderson_cooper_precision_by_bucket = get_precision_by_bucket(anderson_cooper_results_by_bucket)
anderson_cooper_exp_pos_by_bucket = compute_exp_positives(
    anderson_cooper_results_by_bucket, anderson_cooper_precision_by_bucket)
plot_precision_and_cdf(anderson_cooper_precision_by_bucket, anderson_cooper_exp_pos_by_bucket)
save_results('<NAME>', anderson_cooper_precision_by_bucket, anderson_cooper_exp_pos_by_bucket)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1013400852220095, 'fraction_numerical': 0.14111035632680788, 'mean_word_length': 4.517546848381602, 'pattern_counts': {'":': 0, '<': 87, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '246976', 'license': 'Apache-2.0', 'n_tokens_mistral': 7772, 'n_tokens_neox': 6035, 'n_words': 978, 'path': 'app/notebooks/face_search_eval.ipynb', 'repo_name': 'scanner-research/esper-tv', 'size': '16422'}