<commit_before>function inspect(index::Index, match_ind=0)
  meshset = load("MeshSet", index)
  inspect(meshset, match_ind)
end

function inspect(firstindex::Index, lastindex::Index, match_ind=0)
  if is_prealigned(firstindex) && is_prealigned(lastindex)
    k = match_ind
    if match_ind == 0
      k = 1
    end 
    meshset = load_split(get_name(firstindex, lastindex), k)
  else
    meshset = load("MeshSet", (firstindex, lastindex))
  end
  inspect(meshset, match_ind)
end

function inspect(meshset::MeshSet, match_ind=0)
  if match_ind == 0
    meshset, match_ind = get_next_flagged_match(meshset, 0)
  end
  k = match_ind
  name = get_name(meshset)
  num_string = string(" @ ", match_ind, " / ", length(meshset.matches))
  #=if is_prealigned(meshset.meshes[1].index) || is_prealigned(meshset.meshes[2].index)
    k = 1
    name = string("#", get_name(meshset), " ", get_parent(meshset))
    num_string = ""
  end=#
  println("\n", name, ": ", get_name(meshset.matches[k]), num_string)
  imgc, img2, params = view_match(meshset, k)
  enable_inspection(imgc, img2, meshset, k, params)
end

"""
Get next match in the wafer (could span meshsets)
"""
function get_next_match(meshset::MeshSet, match_ind=1)
  firstindex = meshset.meshes[1].index
  lastindex = meshset.meshes[end].index
  if !is_aligned(meshset)
    matches = meshset.matches
    if 1 <= match_ind < length(matches)
      return meshset, match_ind+1
    elseif match_ind == length(matches)
      if is_premontaged(firstindex)
        indexA = premontaged(get_succeeding_in_wafer(montaged(lastindex)))
        indexB = firstindex
      elseif is_montaged(firstindex)
        indexA = get_succeeding_in_wafer(firstindex)
        indexB = firstindex
      end

      if indexA[1:2] == (0,0) || indexB[1:2] == (0,0)
        return nothing, nothing
      else
        meshset = load("MeshSet", (indexA, indexB))
        match_ind = 1
        return meshset, match_ind
      end
    end

  elseif is_aligned(meshset)
    parent_name = get_parent(meshset)
    match_ind += 1
    if match_ind <= count_children(parent_name)
      return load_split(parent_name, match_ind), match_ind
    else
      return nothing, nothing
    end
  end
  return meshset, 1
end

"""
Get previous match in the wafer (could span meshsets)
"""
function get_previous_match(meshset::MeshSet, match_ind=1)
  firstindex = meshset.meshes[1].index
  lastindex = meshset.meshes[end].index
  if !is_aligned(meshset)
    matches = meshset.matches
    if 1 < match_ind <= length(matches)
      return meshset, match_ind-1
    elseif match_ind == 1
      if is_premontaged(firstindex)
        indexA = premontaged(get_preceding_in_wafer(montaged(firstindex)))
        indexB = firstindex
      elseif is_montaged(firstindex)
        indexA = lastindex
        indexB = get_preceding_in_wafer(lastindex)
      end

      if indexA[1:2] == (0,0) || indexB[1:2] == (0,0)
        return nothing, nothing
      else
        meshset = load("MeshSet", (indexA, indexB))
        match_ind = length(meshset.matches)
        return meshset, match_ind
      end
    end

  elseif is_aligned(meshset)
    parent_name = get_parent(meshset)
    match_ind -= 1
    if match_ind > 0
      return load_split(parent_name, match_ind), match_ind
    else
      return nothing, nothing
    end
  end
  return meshset, 1
end

"""
Return next match in the matches list that's flagged
"""
function get_next_flagged_match(meshset::MeshSet, match_ind=1)
  meshset, match_ind = get_next_match(meshset, match_ind)
  if meshset == nothing || match_ind == nothing
    return nothing, nothing
  end
  k = match_ind
  if is_aligned(meshset)
    k = 1
  end
  if is_flagged(meshset.matches[k])
    return meshset, match_ind
  else
    return get_next_flagged_match(meshset, match_ind)
  end
end

"""
Return previous meshset & match index in the matches list that's flagged
"""
function get_previous_flagged_match(meshset::MeshSet, match_ind)
  meshset, match_ind = get_previous_match(meshset, match_ind)
  if meshset == nothing || match_ind == nothing
    return nothing, nothing
  end
  k = match_ind
  if is_aligned(meshset)
    k = 1
  end
  if is_flagged(meshset.matches[k])
    return meshset, match_ind
  else
    return get_previous_flagged_match(meshset, match_ind)
  end
end

"""
Display match index match_ind as overlayed images with points
"""
function view_match(meshset::MeshSet, match_ind)
  match = meshset.matches[match_ind]
  indexA = match.src_index
  indexB = match.dst_index

  path = get_path("review", (indexB, indexA))
  if !isfile(path)
    path = get_path("review", (indexA, indexB))
    if !isfile(path)
      error("NO REVIEW IMAGE CREATED")
    end
  end
  img_orig = h5read(path, "img")
  offset = h5read(path, "offset")
  println("offset: ", offset)
  scale = h5read(path, "scale")

  # Add border to the image for vectors that extend
  pad = 400
  img = zeros(UInt32, size(img_orig,1)+pad*2, size(img_orig,2)+pad*2)
  img[pad:end-pad-1, pad:end-pad-1] = img_orig

  params = deepcopy(meshset.properties["params"]["match"])
  params["offset"] = offset - pad
  params["scale"] = scale
  params["match_index"] = match_ind
  params["vector_scale"] = 40
  params["dist"] = 90
  params["sigma"] = 20
  params["post_matches"] = false # is_prealigned(indexA)
  if is_montaged(meshset)
    params["vector_scale"] = 4
    params["sigma"] = 7
    params["post_matches"] = true
  end
  params["meshset"] = meshset

  if USE_PYPLOT
    try
      view_inspection_statistics(meshset, match_ind)
    catch e
      println("view_inspection_statistics failed:\n$e")
    end
  end

  imgc, img2 = ImageView.view(img, pixelspacing=[1,1])
  # resize(imgc, 400, 600)
  make_vectors!(params)
  show_vectors(imgc, img2, params["vectors"], RGB(0,0,1), RGB(1,0,1))
  update_annotations(imgc, img2, match, params)
  override_xy_label(imgc, img2, params["offset"], params["scale"])

  return imgc, img2, params
end

function resize(imgc, w, h)
  c = canvas(imgc)
  win = Tk.toplevel(c)
  Tk.set_size(win, w, h)
end

"""
Make vectors for display, scaling and offsetting appropriately
"""
function make_vectors!(params)
  meshset = params["meshset"]
  match_ind = params["match_index"]
  scale = params["scale"]
  offset = params["offset"]
  src_points, dst_points, filtered_inds = get_correspondences(meshset, match_ind; globalized=true)
  if params["post_matches"]
    src_points, dst_points, filtered_inds = get_correspondences(meshset, match_ind; globalized=true, use_post=true)
  end
  vectorsA = scale_matches(src_points, scale)
  vectorsB = scale_matches(dst_points, scale)
  vectorsA, vectorsB = offset_points(vectorsA, vectorsB, offset)
  vectorsA, vectorsB = transpose_points(vectorsA), transpose_points(vectorsB)
  vectors = [hcat(vectorsA...); hcat(vectorsB...)]
  params["displacements"] = vectors
  params["vectors"] = vectors
  change_vector_lengths!(params)
end

"""
Scale a list of 2-element points
"""
function scale_matches(pts, scale)
  return [x*scale for x in pts]
end

"""
Maintain vector start point, but adjust end point for more prominent visual
"""
function change_vector_lengths!(params)
  k = params["vector_scale"]
  vectors = params["displacements"]
  params["vectors"] = []
  if length(vectors) > 0
    params["vectors"] = change_vector_lengths(vectors, k)
  end
end

function change_vector_lengths(vectors, scale)
  return [vectors[1,:]; 
            vectors[2,:]; 
            (vectors[3,:]-vectors[1,:])*scale + vectors[1,:]; 
            (vectors[4,:]-vectors[2,:])*scale + vectors[2,:]]
end

function offset_points(src_pts, dst_pts, offset)
  src_pts = [x - offset for x in src_pts]
  dst_pts = [x - offset for x in dst_pts]
  return src_pts, dst_pts
end

function transpose_points(point_list)
  return map(reverse, point_list)
end

function transpose_vectors(vectors)
  return [vectors[2,:]; vectors[1,:]; vectors[4,:]; vectors[3,:]]
end

function update_annotations(imgc, img2, match, params)
  v = params["vectors"] 
  # v = change_vector_lengths(params["vectors"], params["vector_scale"])
  mask = get_filtered_indices(match)
  for an in Base.values(imgc.annotations)
    if :pts in fieldnames(an.data)
      an.data.pts = v[1:2, mask]
    elseif :lines in fieldnames(an.data)
      an.data.lines = v[:, mask]
    end
  end
  ImageView.redraw(imgc)
end

function view_sigma(match, match_ind)
  src_patch, src_pt, dst_patch, dst_pt, xc, offset = get_correspondence_patches(match, match_ind)
  N=size(xc, 1)
  M=size(xc, 2)

  x = linspace(0, N, N)
  y = linspace(0, M, M)
  xgrid = repmat(x, 1, M)
  ygrid = repmat(y', N, 1)

  fig = figure("sigma_filter_100") #,figsize=(10,10))
  PyPlot.clf()
  ax = fig[:add_subplot](1,1,1, projection = "3d") 
  ax[:plot_surface](xgrid, ygrid, xc, rstride=10, edgecolors="k", 
                    cstride=10, cmap=ColorMap("hot"), alpha=0.8, linewidth=0,
                    antialiased=false) 
  grid("on")
  title("match $match_ind")

  # s = sigma(xc)
  # beta = 0.5
  # xc_beta = xc
  # xc_beta[xc .< beta*maximum(xc)] = NaN
  # ax = fig[:add_subplot](2,1,2, projection = "3d") 
  # ax[:plot_surface](xgrid, ygrid, xc_beta, rstride=10, edgecolors="k", 
  #                   cstride=10, cmap=ColorMap("hot"), alpha=0.8, linewidth=0,
  #                   antialiased=false) 
  # grid("on")
  # title("correlogram")
  # title("beta_threshold")
end

"""
Display the images involved in the blockmatching at one point in a Match object
"""
function view_blockmatch(match, match_ind, params)
  for (k, v) in match.correspondence_properties[match_ind]
    println(k, ":\t", v)
  end
  src_patch, src_pt, dst_patch, dst_pt, xc, offset = get_correspondence_patches(match, match_ind)
  block_r = params["block_r"]
  search_r = params["search_r"]
  beta = 0.5
  N=size(xc, 1)<commit_msg>Partion view_sigma into xcsurface call (like xcview)<commit_after>function inspect(index::Index, match_ind=0)
  meshset = load("MeshSet", index)
  inspect(meshset, match_ind)
end

function inspect(firstindex::Index, lastindex::Index, match_ind=0)
  if is_prealigned(firstindex) && is_prealigned(lastindex)
    k = match_ind
    if match_ind == 0
      k = 1
    end 
    meshset = load_split(get_name(firstindex, lastindex), k)
  else
    meshset = load("MeshSet", (firstindex, lastindex))
  end
  inspect(meshset, match_ind)
end

function inspect(meshset::MeshSet, match_ind=0)
  if match_ind == 0
    meshset, match_ind = get_next_flagged_match(meshset, 0)
  end
  k = match_ind
  name = get_name(meshset)
  num_string = string(" @ ", match_ind, " / ", length(meshset.matches))
  #=if is_prealigned(meshset.meshes[1].index) || is_prealigned(meshset.meshes[2].index)
    k = 1
    name = string("#", get_name(meshset), " ", get_parent(meshset))
    num_string = ""
  end=#
  println("\n", name, ": ", get_name(meshset.matches[k]), num_string)
  imgc, img2, params = view_match(meshset, k)
  enable_inspection(imgc, img2, meshset, k, params)
end

"""
Get next match in the wafer (could span meshsets)
"""
function get_next_match(meshset::MeshSet, match_ind=1)
  firstindex = meshset.meshes[1].index
  lastindex = meshset.meshes[end].index
  if !is_aligned(meshset)
    matches = meshset.matches
    if 1 <= match_ind < length(matches)
      return meshset, match_ind+1
    elseif match_ind == length(matches)
      if is_premontaged(firstindex)
        indexA = premontaged(get_succeeding_in_wafer(montaged(lastindex)))
        indexB = firstindex
      elseif is_montaged(firstindex)
        indexA = get_succeeding_in_wafer(firstindex)
        indexB = firstindex
      end

      if indexA[1:2] == (0,0) || indexB[1:2] == (0,0)
        return nothing, nothing
      else
        meshset = load("MeshSet", (indexA, indexB))
        match_ind = 1
        return meshset, match_ind
      end
    end

  elseif is_aligned(meshset)
    parent_name = get_parent(meshset)
    match_ind += 1
    if match_ind <= count_children(parent_name)
      return load_split(parent_name, match_ind), match_ind
    else
      return nothing, nothing
    end
  end
  return meshset, 1
end

"""
Get previous match in the wafer (could span meshsets)
"""
function get_previous_match(meshset::MeshSet, match_ind=1)
  firstindex = meshset.meshes[1].index
  lastindex = meshset.meshes[end].index
  if !is_aligned(meshset)
    matches = meshset.matches
    if 1 < match_ind <= length(matches)
      return meshset, match_ind-1
    elseif match_ind == 1
      if is_premontaged(firstindex)
        indexA = premontaged(get_preceding_in_wafer(montaged(firstindex)))
        indexB = firstindex
      elseif is_montaged(firstindex)
        indexA = lastindex
        indexB = get_preceding_in_wafer(lastindex)
      end

      if indexA[1:2] == (0,0) || indexB[1:2] == (0,0)
        return nothing, nothing
      else
        meshset = load("MeshSet", (indexA, indexB))
        match_ind = length(meshset.matches)
        return meshset, match_ind
      end
    end

  elseif is_aligned(meshset)
    parent_name = get_parent(meshset)
    match_ind -= 1
    if match_ind > 0
      return load_split(parent_name, match_ind), match_ind
    else
      return nothing, nothing
    end
  end
  return meshset, 1
end

"""
Return next match in the matches list that's flagged
"""
function get_next_flagged_match(meshset::MeshSet, match_ind=1)
  meshset, match_ind = get_next_match(meshset, match_ind)
  if meshset == nothing || match_ind == nothing
    return nothing, nothing
  end
  k = match_ind
  if is_aligned(meshset)
    k = 1
  end
  if is_flagged(meshset.matches[k])
    return meshset, match_ind
  else
    return get_next_flagged_match(meshset, match_ind)
  end
end

"""
Return previous meshset & match index in the matches list that's flagged
"""
function get_previous_flagged_match(meshset::MeshSet, match_ind)
  meshset, match_ind = get_previous_match(meshset, match_ind)
  if meshset == nothing || match_ind == nothing
    return nothing, nothing
  end
  k = match_ind
  if is_aligned(meshset)
    k = 1
  end
  if is_flagged(meshset.matches[k])
    return meshset, match_ind
  else
    return get_previous_flagged_match(meshset, match_ind)
  end
end

"""
Display match index match_ind as overlayed images with points
"""
function view_match(meshset::MeshSet, match_ind)
  match = meshset.matches[match_ind]
  indexA = match.src_index
  indexB = match.dst_index

  path = get_path("review", (indexB, indexA))
  if !isfile(path)
    path = get_path("review", (indexA, indexB))
    if !isfile(path)
      error("NO REVIEW IMAGE CREATED")
    end
  end
  img_orig = h5read(path, "img")
  offset = h5read(path, "offset")
  println("offset: ", offset)
  scale = h5read(path, "scale")

  # Add border to the image for vectors that extend
  pad = 400
  img = zeros(UInt32, size(img_orig,1)+pad*2, size(img_orig,2)+pad*2)
  img[pad:end-pad-1, pad:end-pad-1] = img_orig

  params = deepcopy(meshset.properties["params"]["match"])
  params["offset"] = offset - pad
  params["scale"] = scale
  params["match_index"] = match_ind
  params["vector_scale"] = 40
  params["dist"] = 90
  params["sigma"] = 20
  params["post_matches"] = false # is_prealigned(indexA)
  if is_montaged(meshset)
    params["vector_scale"] = 4
    params["sigma"] = 7
    params["post_matches"] = true
  end
  params["meshset"] = meshset

  if USE_PYPLOT
    try
      view_inspection_statistics(meshset, match_ind)
    catch e
      println("view_inspection_statistics failed:\n$e")
    end
  end

  imgc, img2 = ImageView.view(img, pixelspacing=[1,1])
  # resize(imgc, 400, 600)
  make_vectors!(params)
  show_vectors(imgc, img2, params["vectors"], RGB(0,0,1), RGB(1,0,1))
  update_annotations(imgc, img2, match, params)
  override_xy_label(imgc, img2, params["offset"], params["scale"])

  return imgc, img2, params
end

function resize(imgc, w, h)
  c = canvas(imgc)
  win = Tk.toplevel(c)
  Tk.set_size(win, w, h)
end

"""
Make vectors for display, scaling and offsetting appropriately
"""
function make_vectors!(params)
  meshset = params["meshset"]
  match_ind = params["match_index"]
  scale = params["scale"]
  offset = params["offset"]
  src_points, dst_points, filtered_inds = get_correspondences(meshset, match_ind; globalized=true)
  if params["post_matches"]
    src_points, dst_points, filtered_inds = get_correspondences(meshset, match_ind; globalized=true, use_post=true)
  end
  vectorsA = scale_matches(src_points, scale)
  vectorsB = scale_matches(dst_points, scale)
  vectorsA, vectorsB = offset_points(vectorsA, vectorsB, offset)
  vectorsA, vectorsB = transpose_points(vectorsA), transpose_points(vectorsB)
  vectors = [hcat(vectorsA...); hcat(vectorsB...)]
  params["displacements"] = vectors
  params["vectors"] = vectors
  change_vector_lengths!(params)
end

"""
Scale a list of 2-element points
"""
function scale_matches(pts, scale)
  return [x*scale for x in pts]
end

"""
Maintain vector start point, but adjust end point for more prominent visual
"""
function change_vector_lengths!(params)
  k = params["vector_scale"]
  vectors = params["displacements"]
  params["vectors"] = []
  if length(vectors) > 0
    params["vectors"] = change_vector_lengths(vectors, k)
  end
end

function change_vector_lengths(vectors, scale)
  return [vectors[1,:]; 
            vectors[2,:]; 
            (vectors[3,:]-vectors[1,:])*scale + vectors[1,:]; 
            (vectors[4,:]-vectors[2,:])*scale + vectors[2,:]]
end

function offset_points(src_pts, dst_pts, offset)
  src_pts = [x - offset for x in src_pts]
  dst_pts = [x - offset for x in dst_pts]
  return src_pts, dst_pts
end

function transpose_points(point_list)
  return map(reverse, point_list)
end

function transpose_vectors(vectors)
  return [vectors[2,:]; vectors[1,:]; vectors[4,:]; vectors[3,:]]
end

function update_annotations(imgc, img2, match, params)
  v = params["vectors"] 
  # v = change_vector_lengths(params["vectors"], params["vector_scale"])
  mask = get_filtered_indices(match)
  for an in Base.values(imgc.annotations)
    if :pts in fieldnames(an.data)
      an.data.pts = v[1:2, mask]
    elseif :lines in fieldnames(an.data)
      an.data.lines = v[:, mask]
    end
  end
  ImageView.redraw(imgc)
end

function view_sigma(match, match_ind)
  src_patch, src_pt, dst_patch, dst_pt, xc, offset = get_correspondence_patches(match, match_ind)
  xcsurface(xc)
  grid("on")
  title("match $match_ind")
end

function xcsurface(xc)
  N=size(xc, 1)
  M=size(xc, 2)

  x = linspace(0, N, N)
  y = linspace(0, M, M)
  xgrid = repmat(x, 1, M)
  ygrid = repmat(y', N, 1)

  fig = figure("xc") #,figsize=(10,10))
  PyPlot.clf()
  ax = fig[:add_subplot](1,1,1, projection = "3d") 
  ax[:plot_surface](xgrid, ygrid, xc, rstride=10, edgecolors="k", 
                    cstride=10, cmap=ColorMap("hot"), alpha=0.8, linewidth=0,
                    antialiased=false)
end

"""
Display the images involved in the blockmatching at one point in a Match object
"""
function view_blockmatch(match, match_ind, params)
  for (k, v) in match.correspondence_properties[match_ind]
    println(k, ":\t", v)
  end
  src_patch, src_pt, dst_patch, dst_pt, xc, offset = get_correspondence_patches(match, match_ind)
  block_r = params["block_r"]
  search_r = params["search_r"]
  beta = 0.5
  N=size(xc, 1)
{'fraction_non_alphanumeric': 0.12308896425776154, 'fraction_numerical': 0.014192538481607096, 'mean_word_length': 4.017277486910995, 'pattern_counts': {'":': 4, '<': 14, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '863323', 'n_tokens_mistral': 6968, 'n_tokens_neox': 6722, 'n_words': 1561}