<jupyter_start><jupyter_text>Get all the overlap_image_maps that are not in dup_truth.<jupyter_code>overlap_candidates = []
for img1_id, img2_id, img1_overlap_tag in tqdm_notebook(overlap_image_maps):
    if (img1_id, img2_id, img1_overlap_tag) in dup_truth:
        continue
    overlap_candidates.append((img1_id, img2_id, img1_overlap_tag))
print(len(overlap_candidates))<jupyter_output><empty_output><jupyter_text>Create list of flat hashes. (i.e. hashes for tiles where every pixel is the same color)<jupyter_code>solid_hashes = set()
for img_id, tile_issolid_grid in sdcic.img_metrics['sol'].items():
    idxs = set(np.where(tile_issolid_grid >= 0)[0])
    for idx in idxs:
        if np.all(tile_issolid_grid[idx] >= 0):
            solid_hashes.add(sdcic.img_metrics['md5'][img_id][idx])

print(solid_hashes)<jupyter_output><empty_output><jupyter_text>Automatically find dups using dicts<jupyter_code>tile_hash_dup_dict = defaultdict(set)
tile_hash_dif_dict = defaultdict(set)

for (img1_id, img2_id, img1_overlap_tag), is_dup in dup_truth.items():
    
    for idx1, idx2 in tag_pair_lookup[img1_overlap_tag]:
        
        tile1_hash = sdcic.img_metrics['md5'][img1_id][idx1]
        tile2_hash = sdcic.img_metrics['md5'][img2_id][idx2]
        
        if is_dup:

            if tile1_hash in solid_hashes or tile2_hash in solid_hashes:
                continue

            tile_hash_dup_dict[tile1_hash].add(tile1_hash)
            tile_hash_dup_dict[tile2_hash].add(tile2_hash)
            tile_hash_dup_dict[tile1_hash].add(tile2_hash)
            tile_hash_dup_dict[tile2_hash].add(tile1_hash)
        
        else:
            if tile1_hash == tile2_hash:
                continue

            tile_hash_dif_dict[tile1_hash].add(tile2_hash)
            tile_hash_dif_dict[tile2_hash].add(tile1_hash)
            
print(len(tile_hash_dup_dict), len(tile_hash_dif_dict))

# Sanity check: hashes cannot be simultaneously "a dup" and "not a dup" of tile1_hash
for tile1_hash in tile_hash_dup_dict:
    if len(tile_hash_dup_dict[tile1_hash].intersection(tile_hash_dif_dict[tile1_hash])) != 0:
        print(tile1_hash, tile_hash_dup_dict[tile1_hash], tile_hash_dif_dict[tile1_hash])
    assert len(tile_hash_dup_dict[tile1_hash].intersection(tile_hash_dif_dict[tile1_hash])) == 0
    
# Sanity check: If B and C are dups of A, then make sure C not in tile_hash_dif_dict[B]
for tile1_hash, tile1_dups in tile_hash_dup_dict.items():
    for tile1_dup1 in sorted(tile1_dups):
        for tile1_dup2 in sorted(tile1_dups):
            if tile1_dup1 in tile_hash_dif_dict[tile1_dup2]:
                print(tile1_hash, tile1_dup1, tile_hash_dif_dict[tile1_dup2])
            assert tile1_dup1 not in tile_hash_dif_dict[tile1_dup2]

# Now we should be able to form cliques: (i.e. If A == B and B == C, then A == C)
for tile1_hash, tile1_dups in tile_hash_dup_dict.items():
    for tile1_dup1 in sorted(tile1_dups):
        for tile1_dup2 in sorted(tile1_dups):
            if tile1_dup1 <= tile1_dup2:
                continue
            tile_hash_dup_dict[tile1_dup1].add(tile1_dup2)
            tile_hash_dup_dict[tile1_dup2].add(tile1_dup1)

neighbor_counts = Counter()
for tile1_hash, tile1_dups in tile_hash_dup_dict.items():
    neighbor_counts[len(tile1_dups)] += 1
list(sorted(neighbor_counts.items()))
auto_overlap_labels_0 = {}

for img1_id, img2_id, img1_overlap_tag in overlap_candidates:
    for idx1, idx2 in tag_pair_lookup[img1_overlap_tag]:
        
        tile1_hash = sdcic.img_metrics['md5'][img1_id][idx1]
        tile2_hash = sdcic.img_metrics['md5'][img2_id][idx2]
        
        if tile1_hash in tile_hash_dif_dict[tile2_hash]:
            assert tile2_hash in tile_hash_dif_dict[tile1_hash]
            auto_overlap_labels_0[(img1_id, img2_id, img1_overlap_tag)] = 0
            break

print(len(auto_overlap_labels_0))
auto_overlap_labels_1 = {}

for img1_id, img2_id, img1_overlap_tag in overlap_candidates:
    for idx1, idx2 in tag_pair_lookup[img1_overlap_tag]:
        
        tile1_hash = sdcic.img_metrics['md5'][img1_id][idx1]
        tile2_hash = sdcic.img_metrics['md5'][img2_id][idx2]
        
        if tile1_hash in tile_hash_dup_dict[tile2_hash]:
            assert tile2_hash in tile_hash_dup_dict[tile1_hash]
            continue
        else:
            break
    else:
        auto_overlap_labels_1[(img1_id, img2_id, img1_overlap_tag)] = 1

print(len(auto_overlap_labels_1))<jupyter_output><empty_output><jupyter_text>Using cliques (networkx)<jupyter_code>tile_hash_dup_cliques = nx.Graph()
tile_hash_dif_cliques = nx.Graph()

for (img1_id, img2_id, img1_overlap_tag), is_dup in dup_truth.items():
    for idx1, idx2 in tag_pair_lookup[img1_overlap_tag]:
        tile1_hash = sdcic.img_metrics['md5'][img1_id][idx1]
        tile2_hash = sdcic.img_metrics['md5'][img2_id][idx2]
        if is_dup:
            if tile1_hash in solid_hashes or tile2_hash in solid_hashes:
                continue
            update_tile_cliques(tile_hash_dup_cliques, tile1_hash, tile2_hash)
        else:
            if tile1_hash == tile2_hash:
                continue
            tile_hash_dif_cliques.add_edge(tile1_hash, tile2_hash)

print(tile_hash_dup_cliques.number_of_nodes(), tile_hash_dif_cliques.number_of_nodes())

neighbor_counts = Counter()
for tile_hashes in nx.connected_components(tile_hash_dup_cliques):
    neighbor_counts[len(tile_hashes)] += 1
list(sorted(neighbor_counts.items()))<jupyter_output><empty_output><jupyter_text>Separately<jupyter_code>auto_overlap_labels_0 = {}

for img1_id, img2_id, img1_overlap_tag in overlap_candidates:
    for idx1, idx2 in tag_pair_lookup[img1_overlap_tag]:
        
        tile1_hash = sdcic.img_metrics['md5'][img1_id][idx1]
        tile2_hash = sdcic.img_metrics['md5'][img2_id][idx2]
        
        if tile1_hash in tile_hash_dif_cliques and tile2_hash in set(nx.neighbors(tile_hash_dif_cliques, tile1_hash)):
            auto_overlap_labels_0[(img1_id, img2_id, img1_overlap_tag)] = 0
            break

print(len(auto_overlap_labels_0))
auto_overlap_labels_1 = {}

for img1_id, img2_id, img1_overlap_tag in overlap_candidates:
    for idx1, idx2 in tag_pair_lookup[img1_overlap_tag]:
        
        tile1_hash = sdcic.img_metrics['md5'][img1_id][idx1]
        tile2_hash = sdcic.img_metrics['md5'][img2_id][idx2]
        
        if tile1_hash in tile_hash_dup_cliques and tile2_hash in set(nx.neighbors(tile_hash_dup_cliques, tile1_hash)):
            continue
        else:
            break
    else:
        auto_overlap_labels_1[(img1_id, img2_id, img1_overlap_tag)] = 1

print(len(auto_overlap_labels_1))
auto_overlap_labels = {}
for key in auto_overlap_labels_0:
    assert key not in auto_overlap_labels_1
auto_overlap_labels.update(auto_overlap_labels_0)
auto_overlap_labels.update(auto_overlap_labels_1)
print(len(auto_overlap_labels))<jupyter_output><empty_output><jupyter_text>Simultaneously<jupyter_code>auto_overlap_labels = {}

for img1_id, img2_id, img1_overlap_tag in overlap_candidates:
    if (img1_id, img2_id, img1_overlap_tag) in auto_overlap_labels:
        continue
    is_dup = 1
    for idx1, idx2 in tag_pair_lookup[img1_overlap_tag]:
        
        tile1_hash = sdcic.img_metrics['md5'][img1_id][idx1]
        tile2_hash = sdcic.img_metrics['md5'][img2_id][idx2]
        
        if tile1_hash in tile_hash_dif_cliques and tile2_hash in set(nx.neighbors(tile_hash_dif_cliques, tile1_hash)):
            is_dup = 0
            break
        elif tile1_hash in tile_hash_dup_cliques and tile2_hash in set(nx.neighbors(tile_hash_dup_cliques, tile1_hash)):
            continue
        else:
            is_dup = -1

    if is_dup == -1:
        continue
    
    auto_overlap_labels[(img1_id, img2_id, img1_overlap_tag)] = is_dup

print(len(auto_overlap_labels))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08832163369495852, 'fraction_numerical': 0.03241863433312061, 'mean_word_length': 3.8310727496917387, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '45301', 'license': 'MIT', 'n_tokens_mistral': 3007, 'n_tokens_neox': 2980, 'n_words': 474, 'path': 'notebooks/eda/tile_dicts_vs_cliques.ipynb', 'repo_name': 'WillieMaddox/Airbus_SDC_dup', 'size': '7835'}