<jupyter_start><jupyter_text>Demo of finding correspondences- Demos generating correspondences in PyTorch- Simple timing experiments in pytorch- Demos generating non-correspondences  Find pixel correspondences in PyTorch<jupyter_code>%matplotlib inline
import dense_correspondence_manipulation.utils.utils as utils
utils.add_dense_correspondence_to_python_path()
import correspondence_plotter
from dense_correspondence.dataset.spartan_dataset_masked import SpartanDataset
import os
import numpy as np


config_filename = os.path.join(utils.getDenseCorrespondenceSourceDir(), 'config', 'dense_correspondence', 
                               'dataset', 'composite', 'caterpillar_only.yaml')
config = utils.getDictFromYamlFilename(config_filename)
dataset = SpartanDataset(config=config)

import correspondence_finder
import time
uv_a = (300,200)

scene = "2018-04-10-16-02-59"
img_a_index = dataset.get_random_image_index(scene)
img_a_rgb, img_a_depth, _, img_a_pose = dataset.get_rgbd_mask_pose(scene, img_a_index)

img_b_index = dataset.get_img_idx_with_different_pose(scene, img_a_pose, num_attempts=50)
img_b_rgb, img_b_depth, _, img_b_pose = dataset.get_rgbd_mask_pose(scene, img_b_index)

img_a_depth_numpy = np.asarray(img_a_depth)
img_b_depth_numpy = np.asarray(img_b_depth)

start = time.time()
uv_a = (300,200)
uv_a, uv_b = correspondence_finder.batch_find_pixel_correspondences(img_a_depth_numpy, img_a_pose, 
                                                                    img_b_depth_numpy, img_b_pose,
                                                                    uv_a=uv_a)
print time.time() - start, "seconds"
if uv_a is not None:
    correspondence_plotter.plot_correspondences_direct(img_a_rgb, img_a_depth_numpy, img_b_rgb, img_b_depth_numpy, uv_a, uv_b)
else:
    print "try running this cell again, did not find a correspondence for this pixel"<jupyter_output><empty_output><jupyter_text>PyTorch implementation -- finding many correspondencesNote that in this example, about 1/10 get pruned due to either:1. No depth measurement in image a2. Reprojection is outside FOV of image b3. Occluded: the point from image a is occluded in image b4. No depth measurement in image b (so can't be sure if occluded or not)<jupyter_code>start = time.time()
num_attempts = 50
uv_a, uv_b = correspondence_finder.batch_find_pixel_correspondences(img_a_depth_numpy, img_a_pose, 
                                                                    img_b_depth_numpy, img_b_pose, 
                                                                    num_attempts=num_attempts)
print time.time() - start, "seconds"
print "num attempted: ", num_attempts
print "num valid:     ", len(uv_a[0])
if uv_a is not None:
    correspondence_plotter.plot_correspondences_direct(img_a_rgb, img_a_depth_numpy, img_b_rgb, img_b_depth_numpy, uv_a, uv_b)<jupyter_output><empty_output><jupyter_text>On CPU, how fast can we do many, many samples? (many samples, let's not plot)<jupyter_code>num_attempts = 50000

img_a_index = dataset.get_random_image_index(scene)
img_a_rgb, img_a_depth, _, img_a_pose = dataset.get_rgbd_mask_pose(scene, img_a_index)

img_b_index = dataset.get_img_idx_with_different_pose(scene, img_a_pose, num_attempts=50)
img_b_rgb, img_b_depth, _, img_b_pose = dataset.get_rgbd_mask_pose(scene, img_b_index)

img_a_depth_numpy = np.asarray(img_a_depth)
img_b_depth_numpy = np.asarray(img_b_depth)

start = time.time()
uv_a, uv_b = correspondence_finder.batch_find_pixel_correspondences(img_a_depth_numpy, img_a_pose, 
                                                                    img_b_depth_numpy, img_b_pose,
                                                                    num_attempts=num_attempts,
                                                                    device='CPU')

print time.time() - start, "seconds 1st time"
print "num attempted: ", num_attempts
print "num valid:     ", len(uv_a[0])

start = time.time()
uv_a, uv_b = correspondence_finder.batch_find_pixel_correspondences(img_a_depth_numpy, img_a_pose, 
                                                                    img_b_depth_numpy, img_b_pose,
                                                                    num_attempts=num_attempts,
                                                                    device='CPU')

print time.time() - start, "seconds 2nd time"
print "num attempted: ", num_attempts
print "num valid:     ", len(uv_a[0])

img_a_index = dataset.get_random_image_index(scene)
img_a_rgb, img_a_depth, _, img_a_pose = dataset.get_rgbd_mask_pose(scene, img_a_index)

img_b_index = dataset.get_img_idx_with_different_pose(scene, img_a_pose, num_attempts=50)
img_b_rgb, img_b_depth, _, img_b_pose = dataset.get_rgbd_mask_pose(scene, img_b_index)

img_a_depth_numpy = np.asarray(img_a_depth)
img_b_depth_numpy = np.asarray(img_b_depth)

start = time.time()
uv_a, uv_b = correspondence_finder.batch_find_pixel_correspondences(img_a_depth_numpy, img_a_pose, 
                                                                    img_b_depth_numpy, img_b_pose,
                                                                    num_attempts=num_attempts,
                                                                    device='CPU')
print time.time() - start, "seconds on a new image pair"
print "num attempted: ", num_attempts
print "num valid:     ", len(uv_a[0])<jupyter_output><empty_output><jupyter_text>On GPU, how fast?<jupyter_code>num_attempts = 50000

img_a_index = dataset.get_random_image_index(scene)
img_a_rgb, img_a_depth, _, img_a_pose = dataset.get_rgbd_mask_pose(scene, img_a_index)

img_b_index = dataset.get_img_idx_with_different_pose(scene, img_a_pose, num_attempts=50)
img_b_rgb, img_b_depth, _, img_b_pose = dataset.get_rgbd_mask_pose(scene, img_b_index)

img_a_depth_numpy = np.asarray(img_a_depth)
img_b_depth_numpy = np.asarray(img_b_depth)

start = time.time()
uv_a, uv_b = correspondence_finder.batch_find_pixel_correspondences(img_a_depth_numpy, img_a_pose, 
                                                                    img_b_depth_numpy, img_b_pose,
                                                                    num_attempts=num_attempts,
                                                                    device='GPU')

print time.time() - start, "seconds 1st time"
print "num attempted: ", num_attempts
print "num valid:     ", len(uv_a[0])

start = time.time()
uv_a, uv_b = correspondence_finder.batch_find_pixel_correspondences(img_a_depth_numpy, img_a_pose, 
                                                                    img_b_depth_numpy, img_b_pose,
                                                                    num_attempts=num_attempts,
                                                                    device='GPU')

print time.time() - start, "seconds 2nd time"
print "num attempted: ", num_attempts
print "num valid:     ", len(uv_a[0])

img_a_index = dataset.get_random_image_index(scene)
img_a_rgb, img_a_depth, _, img_a_pose = dataset.get_rgbd_mask_pose(scene, img_a_index)

img_b_index = dataset.get_img_idx_with_different_pose(scene, img_a_pose, num_attempts=50)
img_b_rgb, img_b_depth, _, img_b_pose = dataset.get_rgbd_mask_pose(scene, img_b_index)

img_a_depth_numpy = np.asarray(img_a_depth)
img_b_depth_numpy = np.asarray(img_b_depth)

start = time.time()
uv_a, uv_b = correspondence_finder.batch_find_pixel_correspondences(img_a_depth_numpy, img_a_pose, 
                                                                    img_b_depth_numpy, img_b_pose,
                                                                    num_attempts=num_attempts,
                                                                    device='GPU')
print time.time() - start, "seconds on a new image pair"
print "num attempted: ", num_attempts
print "num valid:     ", len(uv_a[0])<jupyter_output><empty_output><jupyter_text>Timing Comments- Since PyTorch has an asynchronous cuda API, the timing numbers for the GPU version are not really valid- In the end, we actually prefer to perform correspondence matching on CPU, since we can have the PyTorch multithreaded dataset loader be performing correspondence matching in parallel with no cost to the GPU, and then the GPU is reserved for just forwards-backwards passes of the network  Non-correspondences<jupyter_code>num_attempts = 5

img_a_index = dataset.get_random_image_index(scene)
img_a_rgb, img_a_depth, _, img_a_pose = dataset.get_rgbd_mask_pose(scene, img_a_index)

img_b_index = dataset.get_img_idx_with_different_pose(scene, img_a_pose, num_attempts=50)
img_b_rgb, img_b_depth, _, img_b_pose = dataset.get_rgbd_mask_pose(scene, img_b_index)

img_a_depth_numpy = np.asarray(img_a_depth)
img_b_depth_numpy = np.asarray(img_b_depth)

start = time.time()
uv_a, uv_b = correspondence_finder.batch_find_pixel_correspondences(img_a_depth_numpy, img_a_pose, 
                                                                    img_b_depth_numpy, img_b_pose,
                                                                    num_attempts=num_attempts,
                                                                    device='CPU')

start = time.time()
uv_b_non_matches = correspondence_finder.create_non_correspondences(uv_b, img_a_depth_numpy.shape, num_non_matches_per_match=10)
print  time.time() - start, "seconds for non-matches"
if uv_b_non_matches is not None:
    print uv_b_non_matches[0].shape

    import torch
    # This just checks to make sure nothing is out of bounds
    print torch.min(uv_b_non_matches[0])
    print torch.min(uv_b_non_matches[1])
    print torch.max(uv_b_non_matches[0])
    print torch.max(uv_b_non_matches[1])
    
    fig, axes = correspondence_plotter.plot_correspondences_direct(img_a_rgb, img_a_depth_numpy, img_b_rgb, img_b_depth_numpy, uv_a, uv_b, show=False)
    uv_a_long = (torch.t(uv_a[0].repeat(3, 1)).contiguous().view(-1,1), torch.t(uv_a[1].repeat(3, 1)).contiguous().view(-1,1))
    uv_b_non_matches_long = (uv_b_non_matches[0].view(-1,1), uv_b_non_matches[1].view(-1,1) )
    correspondence_plotter.plot_correspondences_direct(img_a_rgb, img_a_depth_numpy, img_b_rgb, img_b_depth_numpy, uv_a_long, uv_b_non_matches_long, use_previous_plot=(fig,axes),
                                                  circ_color='r')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.07659574468085106, 'fraction_numerical': 0.008897485493230174, 'mean_word_length': 2.7305194805194803, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 52, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '516367', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3393, 'n_tokens_neox': 3268, 'n_words': 617, 'path': 'dense_correspondence/correspondence_tools/pytorch-finding-correspondences.ipynb', 'repo_name': 'jan-tgk/pytorch-dense-correspondence', 'size': '10340'}