def find_duplicates_all():
    train_paths = glob.glob('../input/petfinder-adoption-prediction/train_images/*-1.jpg')
    train_paths += glob.glob('../input/petfinder-adoption-prediction/train_images/*-2.jpg')
    test_paths = glob.glob('../input/petfinder-adoption-prediction/test_images/*-1.jpg')
    test_paths += glob.glob('../input/petfinder-adoption-prediction/test_images/*-2.jpg')
    (train_petids, train_hashes) = load_image_and_hash(train_paths)
    (test_petids, test_hashes) = load_image_and_hash(test_paths)
    train_hashes = torch.Tensor(train_hashes).cuda()
    test_hashes = torch.Tensor(test_hashes).cuda()
    sims = np.array([((train_hashes[i] == test_hashes).sum(dim=1).cpu().numpy() / 256) for i in range(train_hashes.shape[0])])
    indices1 = np.where((sims > 0.9))
    indices2 = np.where((indices1[0] != indices1[1]))
    petids1 = [train_petids[i] for i in indices1[0][indices2]]
    petids2 = [test_petids[i] for i in indices1[1][indices2]]
    dups = {tuple(sorted([petid1, petid2])): True for (petid1, petid2) in zip(petids1, petids2)}
    logger.info(('found %d duplicates' % len(dups)))
    return dups
def find_duplicates_all():
    train_paths = glob.glob('../input/petfinder-adoption-prediction/train_images/*-1.jpg')
    train_paths += glob.glob('../input/petfinder-adoption-prediction/train_images/*-2.jpg')
    test_paths = glob.glob('../input/petfinder-adoption-prediction/test_images/*-1.jpg')
    test_paths += glob.glob('../input/petfinder-adoption-prediction/test_images/*-2.jpg')
    (train_petids, train_hashes) = load_image_and_hash(train_paths)
    (test_petids, test_hashes) = load_image_and_hash(test_paths)
    train_hashes = torch.Tensor(train_hashes).cuda()
    test_hashes = torch.Tensor(test_hashes).cuda()
    sims = np.array([((train_hashes[i] == test_hashes).sum(dim=1).cpu().numpy() / 256) for i in range(train_hashes.shape[0])])
    indices1 = np.where((sims > 0.9))
    indices2 = np.where((indices1[0] != indices1[1]))
    petids1 = [train_petids[i] for i in indices1[0][indices2]]
    petids2 = [test_petids[i] for i in indices1[1][indices2]]
    dups = {tuple(sorted([petid1, petid2])): True for (petid1, petid2) in zip(petids1, petids2)}
    logger.info(('found %d duplicates' % len(dups)))
    return dups