<jupyter_start><jupyter_text>Detect and describe local features<jupyter_code>DET = importlib.import_module(config.detector)
detector = DET.Model(config, is_training)

if config.use_nms3d:
    print('Apply 3D NMS instead.')
    heatmaps1, det_endpoints = build_multi_scale_deep_detector_3DNMS(config, detector, photos1, reuse=False)
    heatmaps2, det_endpoints2 = build_multi_scale_deep_detector_3DNMS(config, detector, photos2, reuse=True)
else:
    heatmaps1, det_endpoints = build_multi_scale_deep_detector(config, detector, photos1, reuse=False)
    heatmaps2, det_endpoints2 = build_multi_scale_deep_detector(config, detector, photos2, reuse=True)

degree_maps1 = visualize_degree_map(det_endpoints['ori_maps'])
degree_maps2 = visualize_degree_map(det_endpoints2['ori_maps'])
# extract patches
kpts1 = det_endpoints['kpts']
kpts2 = det_endpoints2['kpts']
batch_inds1 = det_endpoints['batch_inds']

kp_patches1 = build_patch_extraction(config, det_endpoints, photos1)
kp_patches2 = build_patch_extraction(config, det_endpoints2, photos2)

# Descriptor
DESC = importlib.import_module(config.descriptor)
descriptor = DESC.Model(config, is_training)
desc_feats1, desc_endpoints = descriptor.build_model(kp_patches1, reuse=False) # [B*K,D]
desc_feats2, _          = descriptor.build_model(kp_patches2, reuse=True) # [B*K,D]
nn_dist, nn_inds2, _, _, _ = nearest_neighbors(desc_feats1, desc_feats2)
kpts2_corr = tf.cast(tf.gather(kpts2, nn_inds2), tf.float32)<jupyter_output><empty_output><jupyter_text>Check with Ground truth<jupyter_code>kpts1_int = tf.cast(kpts1, tf.int32)
heatmaps1w, visible_masks1, xy_maps1to2 = \
    inverse_warp_view_2_to_1(heatmaps2, depths2, depths1, c2Tc1s, 
                            K1=Ks1, K2=Ks2, 
                            inv_thetas1=inv_thetas1, thetas2=thetas2,
                            depth_thresh=config.depth_thresh)
heatmaps2w, visible_masks2, xy_maps2to1 = \
    inverse_warp_view_2_to_1(heatmaps1, depths1, depths2, c1Tc2s, 
                            K1=Ks2, K2=Ks1,
                            inv_thetas1=inv_thetas2, thetas2=thetas1,
                            depth_thresh=config.depth_thresh)
visible_masks1 = visible_masks1 * valid_masks1 # take 'and'
visible_masks2 = visible_masks2 * valid_masks2

kpts2w = batch_gather_keypoints(xy_maps1to2, batch_inds1, kpts1_int)
kpvis2w = batch_gather_keypoints(visible_masks1, batch_inds1, kpts1_int)[:,0] 

match_dist = tf.maximum(tf.cast(tf.reduce_sum(tf.squared_difference(kpts2_corr, kpts2w), axis=1), tf.float32), 1e-6)
match_dist = tf.sqrt(match_dist)
is_match = tf.cast(tf.less_equal(match_dist, config.dist_thresh), tf.float32) * kpvis2w


# SIFT
sift_endpoints = build_competitor_matching_estimation(config, dist_thresh=config.dist_thresh)
tfconfig = tf.ConfigProto()
tfconfig.gpu_options.allow_growth = True 
sess = tf.Session(config=tfconfig)
sess.run(tf.global_variables_initializer())

# load model
saver = tf.train.Saver()
print('Load trained models...')

if os.path.isdir(config.model):
    checkpoint = tf.train.latest_checkpoint(config.model)
    model_dir = config.model
else:
    checkpoint = config.model
    model_dir = os.path.dirname(config.model)

if checkpoint is not None:
    print('Checkpoint', os.path.basename(checkpoint))
    print("[{}] Resuming...".format(time.asctime()))
    saver.restore(sess, checkpoint)
else:
    raise ValueError('Cannot load model from {}'.format(model_dir))    
print('Done.')
feed_dict = {
    is_training: False,
}
fetch_dict = {
    'photos1': photos1,
    'photos2': photos2,
    'rgbs1': rgbs1,
    'rgbs2': rgbs2,
    'kpts1': kpts1,
    'kpts2': kpts2,
    'kpts2_corr': kpts2_corr,
    'is_match': is_match,
    'degree_maps1': degree_maps1,
    'degree_maps2': degree_maps2,
    'xy_maps1to2': xy_maps1to2,
    'visible_masks1': visible_masks1,
}
outs = sess.run(fetch_dict, feed_dict=feed_dict)
outs['photos1'].shape
match_pair = np.concatenate([outs['kpts1'], outs['kpts2_corr']], axis=1)
match_pair = match_pair[outs['is_match']==1]
rgb1 = outs['rgbs1'][0]
rgb2 = outs['rgbs2'][0]
match_pairs = np.concatenate([outs['kpts1'], outs['kpts2_corr']], axis=1)
match_pairs = match_pairs[outs['is_match']==1]

h1, w1 = rgb1.shape[:2]
h2, w2 = rgb2.shape[:2]
canvas = np.zeros((max(h1, h2), w1+w2, 3), dtype=rgb1.dtype)
canvas[:h1, :w1] = rgb1
canvas[:h2, w1:] = rgb2
fig = plt.figure(frameon=False)
fig = plt.imshow(canvas)

xs = match_pairs[:,[0,2]]
xs[:,1] += w1
ys = match_pairs[:,[1,3]]

lw = .25
alpha = 1
sf = 5
lw = .2
markersize = 1

plt.plot(
    xs.T, ys.T,
    alpha=alpha,
    linestyle="-",
    linewidth=lw,
    aa=False,
    marker='.',
    markersize=markersize,
    color=[0.0, 0.67, 0.0],
);
print('#Matches = {}'.format(len(match_pairs)))
plt.subplot(121)
plt.imshow(outs['degree_maps1'][0])
plt.subplot(122)
plt.imshow(outs['degree_maps2'][0])<jupyter_output><empty_output><jupyter_text>SIFT<jupyter_code>sift = cv2.xfeatures2d.SIFT_create(nfeatures=config.top_k, contrastThreshold=1e-5)
sift_in1 = (rgb1[...,0]*255).astype(np.uint8)
sift_in2 = (rgb2[...,0]*255).astype(np.uint8)
kp1, des1 = sift.detectAndCompute(sift_in1,None)
kp2, des2 = sift.detectAndCompute(sift_in2,None)
kp1 = np.array([[kp.pt[0],kp.pt[1]] for kp in kp1])
kp2 = np.array([[kp.pt[0],kp.pt[1]] for kp in kp2])
print('#SIFT keypoints = {}, {}'.format(len(kp1), len(kp2)))
feed_dict = {
    sift_endpoints['feats1_ph']: des1,
    sift_endpoints['feats2_ph']: des2,
    sift_endpoints['kpts1_ph']: kp1,
    sift_endpoints['kpts2_ph']: kp2,
    sift_endpoints['xy_maps1to2_ph']: outs['xy_maps1to2'],
    sift_endpoints['visible_masks1_ph']: outs['visible_masks1']
}
fetch_dict = {
    'kpts2_corr': sift_endpoints['kpts2_corr'],
    'is_match': sift_endpoints['is_match'],
}
outs = sess.run(fetch_dict, feed_dict)
match_pairs = np.concatenate([kp1, outs['kpts2_corr']], axis=1)
match_pairs = match_pairs[outs['is_match']==1]

h1, w1 = rgb1.shape[:2]
h2, w2 = rgb2.shape[:2]
canvas = np.zeros((max(h1, h2), w1+w2, 3), dtype=rgb1.dtype)
canvas[:h1, :w1] = rgb1
canvas[:h2, w1:] = rgb2
fig = plt.figure(frameon=False)
fig = plt.imshow(canvas)

xs = match_pairs[:,[0,2]]
xs[:,1] += w1
ys = match_pairs[:,[1,3]]

lw = .25
alpha = 1
sf = 5
lw = .2
markersize = 1

plt.plot(
    xs.T, ys.T,
    alpha=alpha,
    linestyle="-",
    linewidth=lw,
    aa=False,
    marker='.',
    markersize=markersize,
    color=[0.0, 0.67, 0.0],
);
print('#Matches = {}'.format(len(match_pairs)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15140244847357817, 'fraction_numerical': 0.04447543778087711, 'mean_word_length': 5.077212806026365, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '435556', 'license': 'Apache-2.0', 'n_tokens_mistral': 2779, 'n_tokens_neox': 2670, 'n_words': 400, 'path': 'pyslam/thirdparty/lfnet/notebooks/demo.ipynb', 'repo_name': 'dysdsyd/VO_benchmark', 'size': '6453'}