<commit_before>// Copyright (c) 2014 Bauhaus-Universita<NAME>ar
// This Software is distributed under the Modified BSD License, see license.txt.
//
// Virtual Reality and Visualization Research Group 
// Faculty of Media, Bauhaus-Universitaet Weimar
// http://www.uni-weimar.de/medien/vr

#include <algorithm>
#include <string>
#include <vector>
#include <iostream>

#include <lamure/pvs/pvs_database.h>
#include <lamure/pvs/pvs_utils.h>

#include <lamure/pvs/grid_regular.h>
#include <lamure/pvs/grid_octree.h>
#include <lamure/pvs/grid_octree_hierarchical.h>
#include <lamure/pvs/grid_octree_hierarchical_v2.h>
#include <lamure/pvs/grid_octree_hierarchical_v3.h>

#include <lamure/pvs/grid_optimizer_octree.h>
#include <lamure/pvs/grid_optimizer_octree_hierarchical.h>

#include <boost/program_options.hpp>
#include <boost/filesystem.hpp>

int main(int argc, char** argv)
{
    // Read additional data from input parameters.
    std::string pvs_input_file_path = "";
    std::string pvs_output_file_path = "";
    std::string grid_type = "";
    float optimization_threshold = 1.0f;

    namespace po = boost::program_options;
    namespace fs = boost::filesystem;

    const std::string exec_name = (argc > 0) ? fs::basename(argv[0]) : "";

    putenv((char *)"__GL_SYNC_TO_VBLANK=0");

    std::string resource_file_path = "";

    po::options_description desc("Usage: " + exec_name + " [OPTION]... INPUT\n\n"
                               "Allowed Options");
    desc.add_options()
      ("pvs-file", po::value<std::string>(&pvs_input_file_path), "specify input file of calculated pvs data")
      ("output-file", po::value<std::string>(&pvs_output_file_path), "specify output file of converted visibility data")
      ("gridtype", po::value<std::string>(&grid_type)->default_value("octree"), "specify type of grid to store visibility data ('octree', 'hierarchical', 'hierarchical2', 'hierarchical3')")
      ("optithresh", po::value<float>(&optimization_threshold)->default_value(1.0f), "specify the threshold at which common data are converged. Default is 1.0, which means data must be 100 percent equal.");
      ;

    po::variables_map vm;
    auto parsed_options = po::command_line_parser(argc, argv).options(desc).allow_unregistered().run();
    po::store(parsed_options, vm);
    po::notify(vm);

    if(pvs_input_file_path == "")
    {
        std::cout << "Please specifiy PVS input file path.\n" << desc;
        return 0;
    }

    if(pvs_output_file_path == "")
    {
        std::cout << "Please specifiy PVS output file path.\n" << desc;
        return 0;
    }

    // Load pvs and grid data.
    std::cout << "Start loading grid and visibility data..." << std::endl;

    std::string grid_input_file_path = pvs_input_file_path;
    grid_input_file_path.resize(grid_input_file_path.length() - 3);
    grid_input_file_path += "grid";

    std::fstream file_in;
    file_in.open(grid_input_file_path, std::ios::in);

    if(!file_in.is_open())
    {
        return false;
    }

    // Read the grid file header to identify the grid type.
    std::string input_file_grid_type;
    file_in >> input_file_grid_type;

    lamure::pvs::grid* input_grid = nullptr;

    if(input_file_grid_type == "regular")
    {
        input_grid = new lamure::pvs::grid_regular();
    }
    else
    {
        std::cout << "Input grid must be of regular grid type." << std::endl;
        return 0;
    }

    if(!input_grid->load_grid_from_file(grid_input_file_path))
    {
        // Loading grid file failed.
        delete input_grid;
        input_grid = nullptr;

        std::cout << "Not able to load input grid file." << std::endl;
        return 0;
    }
    
    if(!input_grid->load_visibility_from_file(pvs_input_file_path))
    {
        // Loading visibility data failed.
        delete input_grid;
        input_grid = nullptr;

        std::cout << "Not able to input visibility data." << std::endl;
        return 0;
    }

    std::cout << "Finished loading." << std::endl;

    // Start conversion into different grid type.
    std::cout << "Preparing new grid..." << std::endl;

    lamure::pvs::grid* output_grid = nullptr;

    std::vector<lamure::node_t> ids;
    for(size_t model_index = 0; model_index < input_grid->get_num_models(); ++model_index)
    {
        ids.push_back(input_grid->get_num_nodes(model_index));
    }

    // TODO: check if the input grid uses proper amount of cells to convert into octree.
    int depth = (int)std::round(std::pow(input_grid->get_cell_count(), 1.0/3.0));
    depth = (int)std::sqrt(depth) + 1;

    if(grid_type == "octree")
    {
        output_grid = new lamure::pvs::grid_octree(depth, input_grid->get_size().x, input_grid->get_position_center(), ids);
    }
    else if(grid_type == "hierarchical")
    {
        output_grid = new lamure::pvs::grid_octree_hierarchical(depth, input_grid->get_size().x, input_grid->get_position_center(), ids);
    }
    else if(grid_type == "hierarchical2")
    {
        output_grid = new lamure::pvs::grid_octree_hierarchical_v2(depth, input_grid->get_size().x, input_grid->get_position_center(), ids);
    }
    else if(grid_type == "hierarchical3")
    {
        output_grid = new lamure::pvs::grid_octree_hierarchical_v3(depth, input_grid->get_size().x, input_grid->get_position_center(), ids);
    }
    else
    {
        std::cout << "Invalid output grid type.\n" << desc;<commit_msg>updated grid converter to allow using compressed regular and hierarchical v3 grids as inputs as well as converting into compressed regular grids<commit_after>// Copyright (c) 2014 <NAME>
// This Software is distributed under the Modified BSD License, see license.txt.
//
// Virtual Reality and Visualization Research Group 
// Faculty of Media, Bauhaus-Universitaet Weimar
// http://www.uni-weimar.de/medien/vr

#include <algorithm>
#include <string>
#include <vector>
#include <iostream>

#include <lamure/pvs/pvs_database.h>
#include <lamure/pvs/pvs_utils.h>

#include <lamure/pvs/grid_regular.h>
#include <lamure/pvs/grid_regular_compressed.h>
#include <lamure/pvs/grid_octree.h>
#include <lamure/pvs/grid_octree_hierarchical.h>
#include <lamure/pvs/grid_octree_hierarchical_v2.h>
#include <lamure/pvs/grid_octree_hierarchical_v3.h>

#include <lamure/pvs/grid_optimizer_octree.h>
#include <lamure/pvs/grid_optimizer_octree_hierarchical.h>

#include <boost/program_options.hpp>
#include <boost/filesystem.hpp>

int main(int argc, char** argv)
{
    // Read additional data from input parameters.
    std::string pvs_input_file_path = "";
    std::string pvs_output_file_path = "";
    std::string grid_type = "";
    float optimization_threshold = 1.0f;

    namespace po = boost::program_options;
    namespace fs = boost::filesystem;

    const std::string exec_name = (argc > 0) ? fs::basename(argv[0]) : "";

    putenv((char *)"__GL_SYNC_TO_VBLANK=0");

    std::string resource_file_path = "";

    po::options_description desc("Usage: " + exec_name + " [OPTION]... INPUT\n\n"
                               "Allowed Options");
    desc.add_options()
      ("pvs-file", po::value<std::string>(&pvs_input_file_path), "specify input file of calculated pvs data")
      ("output-file", po::value<std::string>(&pvs_output_file_path), "specify output file of converted visibility data")
      ("gridtype", po::value<std::string>(&grid_type)->default_value("octree"), "specify type of grid to store visibility data ('octree', 'hierarchical', 'hierarchical2', 'hierarchical3')")
      ("optithresh", po::value<float>(&optimization_threshold)->default_value(1.0f), "specify the threshold at which common data are converged. Default is 1.0, which means data must be 100 percent equal.");
      ;

    po::variables_map vm;
    auto parsed_options = po::command_line_parser(argc, argv).options(desc).allow_unregistered().run();
    po::store(parsed_options, vm);
    po::notify(vm);

    if(pvs_input_file_path == "")
    {
        std::cout << "Please specifiy PVS input file path.\n" << desc;
        return 0;
    }

    if(pvs_output_file_path == "")
    {
        std::cout << "Please specifiy PVS output file path.\n" << desc;
        return 0;
    }

    // Load pvs and grid data.
    std::cout << "Start loading grid and visibility data..." << std::endl;

    std::string grid_input_file_path = pvs_input_file_path;
    grid_input_file_path.resize(grid_input_file_path.length() - 3);
    grid_input_file_path += "grid";

    std::fstream file_in;
    file_in.open(grid_input_file_path, std::ios::in);

    if(!file_in.is_open())
    {
        return false;
    }

    // Read the grid file header to identify the grid type.
    std::string input_file_grid_type;
    file_in >> input_file_grid_type;

    lamure::pvs::grid* input_grid = nullptr;

    if(input_file_grid_type == "regular")
    {
        input_grid = new lamure::pvs::grid_regular();
    }
    else if(input_file_grid_type == "regular_compressed")
    {
        input_grid = new lamure::pvs::grid_regular_compressed();
    }
    else if(input_file_grid_type == "octree_hierarchical_v3")
    {
        input_grid = new lamure::pvs::grid_octree_hierarchical_v3();
    }
    else
    {
        std::cout << "Input grid must be of regular, regular compressed or hierarchical v3 grid type." << std::endl;
        return 0;
    }

    if(!input_grid->load_grid_from_file(grid_input_file_path))
    {
        // Loading grid file failed.
        delete input_grid;
        input_grid = nullptr;

        std::cout << "Not able to load input grid file." << std::endl;
        return 0;
    }
    
    if(!input_grid->load_visibility_from_file(pvs_input_file_path))
    {
        // Loading visibility data failed.
        delete input_grid;
        input_grid = nullptr;

        std::cout << "Not able to input visibility data." << std::endl;
        return 0;
    }

    std::cout << "Finished loading." << std::endl;

    // Start conversion into different grid type.
    std::cout << "Preparing new grid..." << std::endl;

    lamure::pvs::grid* output_grid = nullptr;

    std::vector<lamure::node_t> ids;
    for(size_t model_index = 0; model_index < input_grid->get_num_models(); ++model_index)
    {
        ids.push_back(input_grid->get_num_nodes(model_index));
    }

    // TODO: check if the input grid uses proper amount of cells to convert into octree.
    unsigned int cells_per_axis = (int)std::round(std::pow(input_grid->get_cell_count(), 1.0/3.0));
    unsigned int depth = (unsigned int)std::sqrt(cells_per_axis) + 1;

    if(grid_type == "regular_compressed")
    {
        output_grid = new lamure::pvs::grid_regular_compressed(cells_per_axis, input_grid->get_size().x, input_grid->get_position_center(), ids);
    }
    else if(grid_type == "octree")
    {
        output_grid = new lamure::pvs::grid_octree(depth, input_grid->get_size().x, input_grid->get_position_center(), ids);
    }
    else if(grid_type == "hierarchical")
    {
        output_grid = new lamure::pvs::grid_octree_hierarchical(depth, input_grid->get_size().x, input_grid->get_position_center(), ids);
    }
    else if(grid_type == "hierarchical2")
    {
        output_grid = new lamure::pvs::grid_octree_hierarchical_v2(depth, input_grid->get_size().x, input_grid->get_position_center(), ids);
    }
    else
    {
        std::cout << "Invalid output grid type.\n" << desc;
{'fraction_non_alphanumeric': 0.13591033448062836, 'fraction_numerical': 0.006530756332186039, 'mean_word_length': 3.545527476935419, 'pattern_counts': {'":': 0, '<': 120, '<?xml version=': 0, '>': 82, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5758789', 'n_tokens_mistral': 3883, 'n_tokens_neox': 3763, 'n_words': 801}