<jupyter_start><jupyter_text>False Positives<jupyter_code># data_iter = zip(test_ids, test_pred, test_y)
data_iter = zip(train_ids, train_pred, train_y)

fp_ids = [i for i, y_pred, y_true in data_iter
          if y_pred == 1 and y_true == 0]
fp_faces = query_faces(fp_ids)
fp_widget = esper_widget(query_faces_result(fp_faces))
fp_widget
fp_selected = [fp_faces[i]['id'] for i in fp_widget.selected]
fp_selected<jupyter_output><empty_output><jupyter_text>False Negative<jupyter_code># data_iter = zip(test_ids, test_pred, test_y)
data_iter = zip(train_ids, train_pred, train_y)

fn_ids = [i for i, y_pred, y_true in data_iter
          if y_pred == 0 and y_true == 1]
fn_faces = query_faces(fn_ids)
fn_widget = esper_widget(query_faces_result(fn_faces))
fn_widget
fn_selected = [fn_faces[i]['id'] for i in fn_widget.selected]
fn_selected<jupyter_output><empty_output><jupyter_text>Run KNN on Entire Dataset<jupyter_code>RACE_PRED_DIR = '/app/data/race_pred/'
if not os.path.exists(RACE_PRED_DIR):
    os.makedirs(RACE_PRED_DIR)
batch_size = 100000
max_idx = face_embeddings.count()
max_face_id = face_embeddings.ids(max_idx - 1, 1)[0]
# print('Max face id:', max_face_id)
for i in tqdm(range(0, max_face_id, batch_size)):
    batch_path = os.path.join(RACE_PRED_DIR, '{}.json'.format(i))
    if not os.path.exists(batch_path):
        batch_face_ids = list(range(i, i + batch_size))
        batch_face_ids = [
            a for a, b in zip(batch_face_ids, face_embeddings.exists(batch_face_ids))
            if b == True
        ]
        batch_pred = face_embeddings.knn_predict(
            train_ids, train_y, k=15, ids=batch_face_ids)
        with open(batch_path, 'w') as f:
            json.dump(batch_pred, f)<jupyter_output>100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3456/3456 [00:00<00:00, 60597.19it/s]<jupyter_text>Collapse into a single file<jupyter_code>MIN_SCORE_THRESHOLD = 0

with open('/app/data/black_face_ids.csv', 'w') as out_f:
    out_f.write('face_id,score\n')
    count = 0
    for batch_file in tqdm(os.listdir(RACE_PRED_DIR), 
                           desc='Saving batches'):
        batch_path = os.path.join(RACE_PRED_DIR, batch_file)
        with open(batch_path) as batch_f:
            batch_pred = json.load(batch_f)
            for a, b in batch_pred:
                if b > MIN_SCORE_THRESHOLD:
                    out_f.write('{},{}\n'.format(a, round(b, 2)))
                    count += 1
    print('Wrote {} labels'.format(count))<jupyter_output><empty_output><jupyter_text>Save labels to the database<jupyter_code>black_tag, created = Tag.objects.get_or_create(name='black')
if created:
    print('Created:', black_tag.name)
black_labeler, created = Labeler.objects.get_or_create(name='race:knn-black')
if created:
    print('Created:', black_labeler.name)
valid_face_ids = set()
with open('/app/data/pg/query_face.csv') as f:
    for i, line in enumerate(f):
        line = line.strip()
        if line == '' or i == 0:
            continue
        face_id = int(line.split(',', 1)[0])
        valid_face_ids.add(face_id)
print('{} valid face ids'.format(len(valid_face_ids)))
MIN_SCORE_THRESHOLD = 0.3

for batch_file in sorted(os.listdir(RACE_PRED_DIR)):
    batch_path = os.path.join(RACE_PRED_DIR, batch_file)
    batch_tags = []
    already_checked = False
    with open(batch_path) as batch_f:
        batch_pred = json.load(batch_f)
        for a, b in batch_pred:
            if a in valid_face_ids and b >= MIN_SCORE_THRESHOLD:
                if not already_checked:
                    already_checked = True
                    # Check if the batch already exists
                    try:
                        FaceTag.objects.filter(
                            face__id=a, tag=black_tag, labeler=black_labeler
                        ).get()
                        print('Already saved:', batch_file)
                        break
                    except:
                        print('Not yet saved:', batch_file)

                batch_tags.append(
                    FaceTag(face_id=a, score=b, tag=black_tag,
                            labeler=black_labeler))        
    if len(batch_tags) > 0:
        FaceTag.objects.bulk_create(batch_tags)<jupyter_output>Already saved: 0.json
Already saved: 100000.json
Already saved: 1000000.json
Already saved: 10000000.json
Already saved: 100000000.json
Already saved: 100100000.json
Already saved: 100200000.json
Already saved: 100300000.json
Already saved: 100400000.json
Already saved: 100500000.json
Already saved: 100600000.json
Already saved: 100700000.json
Already saved: 100800000.json
Already saved: 100900000.json
{'fraction_non_alphanumeric': 0.129723991507431, 'fraction_numerical': 0.03481953290870488, 'mean_word_length': 3.0787878787878786, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '539634', 'license': 'Apache-2.0', 'n_tokens_mistral': 1865, 'n_tokens_neox': 1638, 'n_words': 328, 'path': 'app/notebooks/race_knn_classifier.ipynb', 'repo_name': 'scanner-research/esper-tv', 'size': '4710'}