<commit_before>// __BEGIN_LICENSE__
//  Copyright (c) 2009-2013, United States Government as represented by the
//  Administrator of the National Aeronautics and Space Administration. All
//  rights reserved.
//
//  The NGT platform is licensed under the Apache License, Version 2.0 (the
//  "License"); you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//  http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
// __END_LICENSE__


/// \file orbitviz.cc
///

/************************************************************************
 *     File: orbitviz.cc
 ************************************************************************/
#include <vw/Core.h>
#include <vw/Math.h>
#include <vw/FileIO/KML.h>
//#include <vw/FileIO/FileUtils.h>
#include <vw/Camera.h>
#include <vw/Cartography.h>

#include <asp/Core/Macros.h>
#include <asp/Core/StereoSettings.h>
#include <asp/Sessions/StereoSessionFactory.h>

#if defined(ASP_HAVE_PKG_ISISIO) && ASP_HAVE_PKG_ISISIO == 1
#include <asp/IsisIO/IsisCameraModel.h>
#endif

#include <iomanip>

#include <boost/algorithm/string.hpp>
#include <boost/program_options.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/convenience.hpp>
#include <boost/filesystem/fstream.hpp>
using namespace vw;
using namespace vw::camera;
using namespace vw::cartography;
namespace po = boost::program_options;
//namespace fs = boost::filesystem;

struct Options : public asp::BaseOptions {
  Options() : seperate_camera_files(true) {}
  // Input
  std::vector<std::string> input_files;
  std::string stereo_session_string, 
              path_to_outside_model, 
              bundle_adjust_prefix;

  // Settings
  bool seperate_camera_files, write_csv, load_camera_solve;
  std::string datum;
  double model_scale; ///< Size scaling applied to 3D models

  // Output
  std::string out_file;
};


// TODO: Update the function in VW
/// Returns the extension of a file.
std::string get_extension( std::string const& input, bool make_lower=true ) {
  boost::filesystem::path ipath( input );
  std::string ext = ipath.extension().string();
  if (make_lower)
    boost::algorithm::to_lower(ext);
  return ext;
}

/// Strip the directory out of a file path
std::string strip_directory( std::string const& input){
 boost::filesystem::path p(input); 
 return p.filename().string();
}

/// Strip the directory and extension out of a file path
std::string strip_directory_and_extension( std::string const& input){
 boost::filesystem::path p(input); 
 return p.stem().string();
}

// Would be nice to have this in a function
/// Populate a list of all files in a directory.
/// - Returns the number of files found.
/// - If an extension is passed in, files must match the extension.
size_t get_files_in_folder(std::string              const& folder,
                           std::vector<std::string>      & output,
                           std::string              const& ext="")
{
  output.clear();
  
  // Handle invalid inputs
  if(!boost::filesystem::exists(folder) || !boost::filesystem::is_directory(folder)) 
    return 0;

  boost::filesystem::directory_iterator it(folder);
  boost::filesystem::directory_iterator endit;

  if (ext != ""){ // Check the extension
    while(it != endit) {
        if(boost::filesystem::is_regular_file(*it) && it->path().extension() == ext) 
          output.push_back(it->path().filename().string());
        ++it;
    }
  }
  else{ // No extension check
    while(it != endit) {
        if(boost::filesystem::is_regular_file(*it)) 
          output.push_back(it->path().filename().string());
        ++it;
    }
  }
  return output.size();
}



// TODO: Eliminate bool input and move somewhere else.
/// Seperates a list of files into camera files and image files.
/// - The camera files may be the same as the image files.
size_t split_files_list(std::vector<std::string> const& input_files,
                        std::vector<std::string>      & image_files, 
                        std::vector<std::string>      & camera_files,
                        const bool seperate_camera_files) {

  size_t num_cameras = seperate_camera_files ? input_files.size()/2 : input_files.size();
  vw_out() << "Number of cameras: " << num_cameras << std::endl;
  
  // Split the list of image files and camera files
  image_files.resize (num_cameras);
  camera_files.resize(num_cameras);
  for (size_t i = 0, store=0; store < num_cameras; ++store) {
    image_files[store] = input_files[i];
    if (seperate_camera_files){ // Camera file comes after the image file
      camera_files[store] = input_files[i+1];
      i += 2;
    } else { // Camera file is the same as the image file
      camera_files[store] = camera_files[store];
      i += 1;
    }
  } // End loop through cameras
  
  return num_cameras;
}

/// Get a list of the files in the solver output folder
size_t get_files_from_solver_folder(std::string                 const& solver_folder,
                                      std::vector<std::string>       & image_files, 
                                      std::vector<std::string>       & camera_files,
                                      std::vector<std::vector<int> > & matched_cameras) {
  image_files.clear();
  camera_files.clear();
  matched_cameras.clear();
  std::vector<std::string> solver_files;
  get_files_in_folder(solver_folder, solver_files);
  
  // Identify the image extension
  std::string image_ext = "";
  for (size_t i=0; i<solver_files.size(); ++i) {
    if (asp::has_image_extension(solver_files[i])) {
      image_ext = get_extension(solver_files[i], false);
      break;
    }
  }
  vw_out() << "image_ext: " << image_ext << std::endl;
  if (image_ext == "")
    vw_throw( ArgumentErr() << "Failed to find any image files in the camera_solve directory: "
                            << solver_folder);
  
  // Grab a list of just the image files
  size_t num_images = get_files_in_folder(solver_folder, image_files, image_ext);
  vw_out() << "Number of cameras: " << num_images << std::endl;
  
  // Make a list of the camera model files
  camera_files.resize(num_images);
  for (size_t i=0; i<num_images; ++i)
    camera_files[i] = solver_folder + "/" + image_files[i] + ".final.tsai";
    
  // Search for IP match files
  std::vector<std::string> match_files;
  size_t num_matches = get_files_in_folder(solver_folder, match_files, ".match");
  matched_cameras.resize(num_images);
  
  // Figure out which image pairs are matched
  std::string f1, f2;
  for (size_t i=0; i<num_images; ++i) {
    matched_cameras[i].clear(); // Start with empty list of matches
    f1 = strip_directory_and_extension(image_files[i]);
    //std::cout <<"f1 = " << f1 << std::endl;
    for (size_t j=0; j<num_images; ++j) {
      if (i == j) // No self-matching!
        continue;
      f2 = strip_directory_and_extension(image_files[j]);
      //std::cout <<"- f2 = " << f2 << std::endl;
      for (size_t m=0; m<num_matches; ++m) {
        // If both file names are contained in the match file name,
        //  make a match between image files i and j
        if ( (match_files[m].find(f1) != std::string::npos) && 
             (match_files[m].find(f2) != std::string::npos)   ) {<commit_msg>Orbitviz now only displays camera_solve matches with a mimimum number of IPs<commit_after>// __BEGIN_LICENSE__
//  Copyright (c) 2009-2013, United States Government as represented by the
//  Administrator of the National Aeronautics and Space Administration. All
//  rights reserved.
//
//  The NGT platform is licensed under the Apache License, Version 2.0 (the
//  "License"); you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//  http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
// __END_LICENSE__


/// \file orbitviz.cc
///

/************************************************************************
 *     File: orbitviz.cc
 ************************************************************************/
#include <vw/Core.h>
#include <vw/Math.h>
#include <vw/FileIO/KML.h>
//#include <vw/FileIO/FileUtils.h>
#include <vw/Camera.h>
#include <vw/Cartography.h>
#include <vw/InterestPoint/InterestData.h>

#include <asp/Core/Macros.h>
#include <asp/Core/StereoSettings.h>
#include <asp/Sessions/StereoSessionFactory.h>

#if defined(ASP_HAVE_PKG_ISISIO) && ASP_HAVE_PKG_ISISIO == 1
#include <asp/IsisIO/IsisCameraModel.h>
#endif

#include <iomanip>

#include <boost/algorithm/string.hpp>
#include <boost/program_options.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/convenience.hpp>
#include <boost/filesystem/fstream.hpp>
using namespace vw;
using namespace vw::camera;
using namespace vw::cartography;
namespace po = boost::program_options;
//namespace fs = boost::filesystem;

struct Options : public asp::BaseOptions {
  Options() : seperate_camera_files(true) {}
  // Input
  std::vector<std::string> input_files;
  std::string stereo_session_string, 
              path_to_outside_model, 
              bundle_adjust_prefix;

  // Settings
  bool seperate_camera_files, write_csv, load_camera_solve;
  std::string datum;
  double model_scale; ///< Size scaling applied to 3D models

  // Output
  std::string out_file;
};


// TODO: Update the function in VW
/// Returns the extension of a file.
std::string get_extension( std::string const& input, bool make_lower=true ) {
  boost::filesystem::path ipath( input );
  std::string ext = ipath.extension().string();
  if (make_lower)
    boost::algorithm::to_lower(ext);
  return ext;
}

/// Strip the directory out of a file path
std::string strip_directory( std::string const& input){
 boost::filesystem::path p(input); 
 return p.filename().string();
}

/// Strip the directory and extension out of a file path
std::string strip_directory_and_extension( std::string const& input){
 boost::filesystem::path p(input); 
 return p.stem().string();
}

// Would be nice to have this in a function
/// Populate a list of all files in a directory.
/// - Returns the number of files found.
/// - If an extension is passed in, files must match the extension.
size_t get_files_in_folder(std::string              const& folder,
                           std::vector<std::string>      & output,
                           std::string              const& ext="")
{
  output.clear();
  
  // Handle invalid inputs
  if(!boost::filesystem::exists(folder) || !boost::filesystem::is_directory(folder)) 
    return 0;

  boost::filesystem::directory_iterator it(folder);
  boost::filesystem::directory_iterator endit;

  if (ext != ""){ // Check the extension
    while(it != endit) {
        if(boost::filesystem::is_regular_file(*it) && it->path().extension() == ext) 
          output.push_back(it->path().filename().string());
        ++it;
    }
  }
  else{ // No extension check
    while(it != endit) {
        if(boost::filesystem::is_regular_file(*it)) 
          output.push_back(it->path().filename().string());
        ++it;
    }
  }
  return output.size();
}



// TODO: Eliminate bool input and move somewhere else.
/// Seperates a list of files into camera files and image files.
/// - The camera files may be the same as the image files.
size_t split_files_list(std::vector<std::string> const& input_files,
                        std::vector<std::string>      & image_files, 
                        std::vector<std::string>      & camera_files,
                        const bool seperate_camera_files) {

  size_t num_cameras = seperate_camera_files ? input_files.size()/2 : input_files.size();
  vw_out() << "Number of cameras: " << num_cameras << std::endl;
  
  // Split the list of image files and camera files
  image_files.resize (num_cameras);
  camera_files.resize(num_cameras);
  for (size_t i = 0, store=0; store < num_cameras; ++store) {
    image_files[store] = input_files[i];
    if (seperate_camera_files){ // Camera file comes after the image file
      camera_files[store] = input_files[i+1];
      i += 2;
    } else { // Camera file is the same as the image file
      camera_files[store] = camera_files[store];
      i += 1;
    }
  } // End loop through cameras
  
  return num_cameras;
}

/// Get a list of the files in the solver output folder
size_t get_files_from_solver_folder(std::string                 const& solver_folder,
                                      std::vector<std::string>       & image_files, 
                                      std::vector<std::string>       & camera_files,
                                      std::vector<std::vector<int> > & matched_cameras) {
  image_files.clear();
  camera_files.clear();
  matched_cameras.clear();
  std::vector<std::string> solver_files;
  get_files_in_folder(solver_folder, solver_files);
  
  // Identify the image extension
  std::string image_ext = "";
  for (size_t i=0; i<solver_files.size(); ++i) {
    if (asp::has_image_extension(solver_files[i])) {
      image_ext = get_extension(solver_files[i], false);
      break;
    }
  }
  vw_out() << "image_ext: " << image_ext << std::endl;
  if (image_ext == "")
    vw_throw( ArgumentErr() << "Failed to find any image files in the camera_solve directory: "
                            << solver_folder);
  
  // Grab a list of just the image files
  size_t num_images = get_files_in_folder(solver_folder, image_files, image_ext);
  vw_out() << "Number of cameras: " << num_images << std::endl;
  
  // Make a list of the camera model files
  camera_files.resize(num_images);
  for (size_t i=0; i<num_images; ++i)
    camera_files[i] = solver_folder + "/" + image_files[i] + ".final.tsai";
    
  // Search for IP match files
  std::vector<std::string> match_files;
  size_t num_matches = get_files_in_folder(solver_folder, match_files, ".match");
  matched_cameras.resize(num_images);
  
  // Ignore any match files where there are not many matches
  const size_t MIN_MATCHES = 30; // This is the default value, but it could be made an option.
  std::vector<ip::InterestPoint> ip1, ip2;
  for (size_t m=0; m<num_matches; ++m) {
    vw::ip::read_binary_match_file(solver_folder+ "/"+match_files[m], ip1, ip2);
    //std::cout << "Read " << ip1.size() << " matches from file " << match_files[m] << std::endl;
    if (ip1.size() < MIN_MATCHES)
      match_files[m] = "";
  }
  
  // Figure out which image pairs are matched
  std::string f1, f2;
  for (size_t i=0; i<num_images; ++i) {
    matched_cameras[i].clear(); // Start with empty list of matches
    f1 = strip_directory_and_extension(image_files[i]);
    //std::cout <<"f1 = " << f1 << std::endl;
    for (size_t j=0; j<num_images; ++j) {
      if (i == j) // No self-matching!
        continue;
      f2 = strip_directory_and_extension(image_files[j]);
      //std::cout <<"- f2 = " << f2 << std::endl;
      for (size_t m=0; m<num_matches; ++m) {
        // If both file names are contained in the match file name,
        //  make a match between image files i and j
        if ( (match_files[m].find(f1) != std::string::npos) && 
             (match_files[m].find(f2) != std::string::npos)   ) {
{'fraction_non_alphanumeric': 0.14631678785962693, 'fraction_numerical': 0.005121719886184002, 'mean_word_length': 3.206382978723404, 'pattern_counts': {'":': 0, '<': 155, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6479812', 'n_tokens_mistral': 5062, 'n_tokens_neox': 4888, 'n_words': 1335}