<commit_before>//
//  meanie3D-detect
//  cf-algorithms
//
//  Created by <NAME> on 5/3/12.
//  Copyright (c) 2012 <NAME>. All rights reserved.
//

#include <boost/tokenizer.hpp>
#include <boost/program_options.hpp>
#include <boost/filesystem.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/exception/info.hpp>
#include <boost/smart_ptr.hpp>

#include <map>
#include <vector>
#include <string>
#include <sstream>
#include <fstream>
#include <exception>
#include <locale>
#include <limits>
#include <stdlib.h>
#include <string>
#include <netcdf>
#include <unistd.h>

#include <meanie3D/meanie3D.h>
#include <radolan/radolan.h>

using namespace std;
using namespace boost;
using namespace netCDF;
using namespace m3D;
using namespace Radolan;

#pragma mark -
#pragma mark type definitions

/** Feature-space data type */
typedef double FS_TYPE;

typedef vector<float> fvec_t;
typedef vector<size_t> bin_t;
typedef vector<string> svec_t;

#pragma mark -
#pragma mark comand line parsing

void parse_commmandline(program_options::variables_map vm,
        string &basename,
        string &sourcepath,
        svec_t &vtk_dim_names,
        bool &create_length_stats,
        bin_t &length_histogram_bins,
        bool &create_speed_stats,
        fvec_t &speed_histogram_bins,
        bool & create_direction_stats,
        fvec_t &direction_histogram_classes,
        bool &create_cluster_stats,
        bin_t &cluster_histogram_bins,
        bool &include_degenerates,
        bool &create_cumulated_size_stats,
        bin_t &size_histogram_bins,
        bool &write_center_tracks_as_vtk,
        bool &write_cumulated_tracks_as_vtk,
        bool &write_gnuplot_files,
        bool &write_track_dictionary)
{
    // Version

    sourcepath = vm["sourcepath"].as<string>();

    if (vm.count("basename") == 0)
    {
        basename = ::m3D::utils::common_component(sourcepath, ".nc");
    } else
    {
        basename = vm["basename"].as<string>();
    }

    write_center_tracks_as_vtk = vm.count("write-center-tracks-as-vtk") > 0;

    write_gnuplot_files = vm.count("write-gnuplot-files") > 0;

    write_cumulated_tracks_as_vtk = vm.count("write-cumulated-tracks-as-vtk") > 0;

    write_track_dictionary = vm.count("write-track-dictionary") > 0;

    include_degenerates = vm["exclude-degenerates"].as<bool>();

    // Default is the dimensions

    if (vm.count("vtk-dimensions") > 0)
    {
        vtk_dim_names = vm["vtk-dimensions"].as<svec_t>();
    }

    create_length_stats = vm.count("create-length-statistics") > 0;
    length_histogram_bins = vm["length-histogram-classes"].as<bin_t>();

    create_speed_stats = vm.count("create-speed-statistics") > 0;
    speed_histogram_bins = vm["speed-histogram-classes"].as<fvec_t>();

    create_direction_stats = vm.count("create-direction-statistics") > 0;
    direction_histogram_classes = vm["direction-histogram-classes"].as<fvec_t>();

    create_cumulated_size_stats = vm.count("create-cumulated-size-statistics") > 0;
    size_histogram_bins = vm["size-histogram-classes"].as<bin_t>();

    create_cluster_stats = vm.count("create-cluster-statistics") > 0;
    cluster_histogram_bins = vm["cluster-histogram-classes"].as<bin_t>();
}

#pragma mark -
#pragma mark histogram helpers

/** updates the size histogram
 * @param classes
 * @param counts
 * @param value
 * @param exceeded_max_class contains <code>true</code> if the
 *        value was added to the 'over' bucket. <code>false</code> else
 */
template <typename T>
void add_value_to_histogram(const vector<T> &classes,
        bin_t &counts,
        T value,
        bool &exceeded_max_class)
{
    bool found_bin = false;

    exceeded_max_class = false;

    for (size_t i = 0; i < classes.size() && !found_bin; i++)
    {
        if (i > 0)
        {
            if (value <= classes[i] && value > classes[i - 1])
            {
                counts[i] = counts[i] + 1;
                found_bin = true;
            }
        } else
        {
            if (value <= classes[0])
            {
                counts[i] = counts[i] + 1;
                found_bin = true;
            }
        }
    }

    // not found? add to the last one

    if (!found_bin)
    {
        // add the last class with value 0 if required

        if (counts.size() == classes.size())
        {
            counts.push_back(0);
            cerr << "WARNING: value bigger than highest histogram class. Adding an 'over' bucket for values higher than " << classes[classes.size() - 1] << endl;
        }

        // count up the 'over' bucket

        counts[classes.size()] = counts[classes.size()] + 1;

        exceeded_max_class = true;
    }
}

/** Histogram output */

template <typename T>
void print_histogram(const vector<T> &classes,
        const bin_t &values,
        ofstream &file)
{
    for (size_t i = 0; i < classes.size(); i++)
    {
        file << classes[i] << "," << values[i] << endl;
        cout << classes[i] << "," << values[i] << endl;
    }

    // 'Over' bucket?

    if (values.size() > classes.size())
    {
        file << " > " << classes[classes.size() - 1] << "," << values.back() << endl;
        cout << " > " << classes[classes.size() - 1] << "," << values.back() << endl;
    }
}

template <typename T>
void write_histogram(const std::string &filename,
        const std::string &x,
        const std::string &y,
        const vector<T> &classes,
        const bin_t &values)
{
    ofstream file(filename.c_str());

    file << "#" << x << "  " << y << endl;

    for (size_t i = 0; i < classes.size(); i++)
    {
        file << classes[i] << "  " << values[i] << endl;
    }

    file.close();
}

template <typename T>
void write_values(const std::string &filename,
        const std::string &x,
        const std::string &y,
        const map<T, size_t> &values)
{
    ofstream file(filename.c_str());

    file << "track length  number of tracks" << endl;

    typename map<T, size_t>::const_iterator si;

    for (si = values.begin(); si != values.end(); si++)
    {
        file << si->first << "  " << si->second << endl;
    }

    file.close();
}

template <typename T>
double average(const vector<FS_TYPE> &v)
{
    // calculate the mean value
    double sum = 0.0;
    for (size_t i = 0; i < v.size(); i++)
    {
        sum += boost::numeric_cast<double>(v[i]);
    }
    return sum / boost::numeric_cast<double>(v.size());
}

/** Takes the given 'distribution' and calculates the weighed average
 * @param distribution map
 * @param If <code>true</code> then the class '1' is ignored. 
 *        If <code>false</code> it counts normally.
 */
template <typename T>
double average(const map<T, size_t> &m, bool ignore_one = true)
{
    // calculate the mean value
    double sum = 0.0;

    size_t total_count = 0;

    typename std::map<T, size_t>::const_iterator mi;

    for (mi = m.begin(); mi != m.end(); mi++)
    {
        T key = mi->first;

        size_t val = mi->second;

        if (key == 1 && ignore_one)
        {
            continue;
        }

        total_count += val;

        sum += boost::numeric_cast<double>(mi->first * mi->second);
    }

    return sum / boost::numeric_cast<double>(total_count);
}

#pragma mark -
#pragma mark Memory management

#if APPLE 
#include <mach/vm_statistics.h>
#include <mach/mach_types.h> 
#include <mach/mach_init.h>
#include <mach/mach_host.h>

/** Calculates available and used memory in megabytes
 * 
 * @param free_memory
 * @param used_memory
 */
void get_memory(double &free_memory, double &used_memory)
{
    const double MB = 1024.0 * 1024.0;
    
    vm_size_t page_size;
    mach_port_t mach_port;
    mach_msg_type_number_t count;
    vm_statistics_data_t vm_stats;

    mach_port = mach_host_self();
    
    count = sizeof(vm_stats) / sizeof(natural_t);
    
    if (KERN_SUCCESS == host_page_size(mach_port, &page_size) &&
        KERN_SUCCESS == host_statistics(mach_port, HOST_VM_INFO, 
                                        (host_info_t)&vm_stats, &count))
    {
        free_memory = ((double)vm_stats.free_count * (int64_t)page_size) / MB;
        
        

        used_memory = (((double)vm_stats.active_count + 
                       (double)vm_stats.inactive_count + 
                       (double)vm_stats.wire_count) *  (double)page_size) / MB;
    }
}
#endif

#pragma mark -
#pragma mark main

/**
 *
 *
 */
int main(int argc, char** argv)
{
    using namespace m3D;
    using namespace m3D::utils::vectors;

    size_t length_hist_default_values[] = {5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100, 105, 110};
    bin_t length_hist_default(length_hist_default_values, length_hist_default_values + 22);

    size_t size_hist_default_values[] = {100, 200, 300, 400, 500, 1000, 2000, 3000, 4000, 5000, 7500, 10000, 15000, 20000, 25000, 50000, 100000};
    bin_t size_hist_default(size_hist_default_values, size_hist_default_values + 17);

    size_t cluster_hist_default_values[] = {10, 50, 100, 500, 1000, 5000, 10000, 50000, 100000, 500000, 10000000};
    bin_t cluster_hist_default(cluster_hist_default_values, cluster_hist_default_values + 11);

    float speed_hist_default_values[] = {2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60};
    fvec_t speed_hist_default(speed_hist_default_values, speed_hist_default_values + 30);

    float direction_hist_default_values[] = {15, 30, 45, 60, 75, 90, 105, 120, 135, 150, 175, 200, 215, 230, 245, 260, 275, 300, 315, 330, 345, 360};
    fvec_t direction_hist_default(direction_hist_default_values, direction_hist_default_values + 22);

    // Declare the supported options.

    program_options::options_description desc("Options");
    desc.add_options()
            ("help,h", "produce help message")
            ("version", "print version information and exit")
            ("basename,b", program_options::value<string>()->implicit_value(""), "Basename for filtering input files. Only files starting with this are used. It is also prepended to result files (optional)")
            ("sourcepath,p", program_options::value<string>()->default_value("."), "Current cluster file (netCDF)")
            ("exclude-degenerates", program_options::value<bool>()->default_value(true), "Exclude results of tracks of length one")
            ("create-length-statistics", "Create a statistic of track lengths.")
            ("length-histogram-classes", program_options::value<bin_t>()->multitoken()->default_value(length_hist_default), "List of track-length values for histogram bins")
            ("create-speed-statistics", "Evaluate speeds of clusters in tracks, based on geometric center point")
            ("speed-histogram-classes", program_options::value<fvec_t>()->multitoken()->default_value(speed_hist_default), "Speed histogram. Values in [m/s]")
            ("create-direction-statistics", "Evaluate directions of clusters in tracks, based on geometric center point")
            ("direction-histogram-classes", program_options::value<fvec_t>()->multitoken()->default_value(direction_hist_default), "Direction histogram. Values in [deg]. Use with radolan grid only!!")
            ("create-cluster-statistics", "Evaluate each cluster in each track in terms of size.")
            ("cluster-histogram-classes", program_options::value<bin_t>()->multitoken()->default_value(cluster_hist_default), "List of cluster size values for histogram bins")
            ("create-cumulated-size-statistics", "Evaluate each track in terms of cumulative size. Warning: this process takes a lot of memory.")
            ("size-histogram-classes", program_options::value<bin_t>()->multitoken()->default_value(size_hist_default), "List of cumulated track size values for histogram bins")
            ("write-track-dictionary", "Write out a dictionary listing tracks with number of clusters etc.")
#if WITH_VTK
            ("write-center-tracks-as-vtk", "Write tracks out as .vtk files")
            ("write-cumulated-tracks-as-vtk", "Write cumulated tracks out as .vtk files. Only has effect if --create-cumulated-size-statistics is used")
            ("vtk-dimensions", program_options::value<svec_t>()->multitoken(), "VTK files are written in the order of dimensions given. This may lead to wrong results if the order of the dimensions is not x,y,z. Add the comma-separated list of dimensions here, in the order you would like them to be written as (x,y,z)")
#endif    
            ("write-gnuplot-files,g", "write individual files for the statistics fit for use with gnuplot")
            ;

    program_options::variables_map vm;

    try
    {
        program_options::store(program_options::parse_command_line(argc, argv, desc), vm);
        program_options::notify(vm);
    } catch (std::exception &e)
    {
        cerr << "FATAL:Error parsing command line: " << e.what() << endl;
        exit(EXIT_FAILURE);
    }

    if (vm.count("help") == 1 || argc < 2)
    {
        cout << desc << "\n";
        return EXIT_SUCCESS;
    }

    if (vm.count("version") != 0)
    {
        cout << m3D::VERSION << endl;
        return EXIT_SUCCESS;<commit_msg>Fixed command line params in trackstats<commit_after>//
//  meanie3D-detect
//  cf-algorithms
//
//  Created by <NAME> on 5/3/12.
//  Copyright (c) 2012 <NAME>. All rights reserved.
//

#include <boost/tokenizer.hpp>
#include <boost/program_options.hpp>
#include <boost/filesystem.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/exception/info.hpp>
#include <boost/smart_ptr.hpp>

#include <map>
#include <vector>
#include <string>
#include <sstream>
#include <fstream>
#include <exception>
#include <locale>
#include <limits>
#include <stdlib.h>
#include <string>
#include <netcdf>
#include <unistd.h>

#include <meanie3D/meanie3D.h>
#include <radolan/radolan.h>

using namespace std;
using namespace boost;
using namespace netCDF;
using namespace m3D;
using namespace Radolan;

#pragma mark -
#pragma mark type definitions

/** Feature-space data type */
typedef double FS_TYPE;

typedef vector<float> fvec_t;
typedef vector<size_t> bin_t;
typedef vector<string> svec_t;

#pragma mark -
#pragma mark comand line parsing

void parse_commmandline(program_options::variables_map vm,
        string &basename,
        string &sourcepath,
        svec_t &vtk_dim_names,
        bool &create_length_stats,
        bin_t &length_histogram_bins,
        bool &create_speed_stats,
        fvec_t &speed_histogram_bins,
        bool & create_direction_stats,
        fvec_t &direction_histogram_classes,
        bool &create_cluster_stats,
        bin_t &cluster_histogram_bins,
        bool &include_degenerates,
        bool &create_cumulated_size_stats,
        bin_t &size_histogram_bins,
        bool &write_center_tracks_as_vtk,
        bool &write_cumulated_tracks_as_vtk,
        bool &write_gnuplot_files,
        bool &write_track_dictionary)
{
    // Version

    sourcepath = vm["sourcepath"].as<string>();

    if (vm.count("basename") == 0)
    {
        basename = ::m3D::utils::common_component(sourcepath, ".nc");
    } else
    {
        basename = vm["basename"].as<string>();
    }

    write_center_tracks_as_vtk = vm.count("write-center-tracks-as-vtk") > 0;

    write_gnuplot_files = vm.count("write-gnuplot-files") > 0;

    write_cumulated_tracks_as_vtk = vm.count("write-cumulated-tracks-as-vtk") > 0;

    write_track_dictionary = vm.count("write-track-dictionary") > 0;

    include_degenerates = vm["exclude-degenerates"].as<bool>();

    // Default is the dimensions

    if (vm.count("vtk-dimensions") > 0)
    {
        vtk_dim_names = vm["vtk-dimensions"].as<svec_t>();
    }

    create_length_stats = vm.count("create-length-statistics") > 0;
    length_histogram_bins = vm["length-histogram-classes"].as<bin_t>();

    create_speed_stats = vm.count("create-speed-statistics") > 0;
    speed_histogram_bins = vm["speed-histogram-classes"].as<fvec_t>();

    create_direction_stats = vm.count("create-direction-statistics") > 0;
    direction_histogram_classes = vm["direction-histogram-classes"].as<fvec_t>();

    create_cumulated_size_stats = vm.count("create-cumulated-size-statistics") > 0;
    size_histogram_bins = vm["size-histogram-classes"].as<bin_t>();

    create_cluster_stats = vm.count("create-cluster-statistics") > 0;
    cluster_histogram_bins = vm["cluster-histogram-classes"].as<bin_t>();
}

#pragma mark -
#pragma mark histogram helpers

/** updates the size histogram
 * @param classes
 * @param counts
 * @param value
 * @param exceeded_max_class contains <code>true</code> if the
 *        value was added to the 'over' bucket. <code>false</code> else
 */
template <typename T>
void add_value_to_histogram(const vector<T> &classes,
        bin_t &counts,
        T value,
        bool &exceeded_max_class)
{
    bool found_bin = false;

    exceeded_max_class = false;

    for (size_t i = 0; i < classes.size() && !found_bin; i++)
    {
        if (i > 0)
        {
            if (value <= classes[i] && value > classes[i - 1])
            {
                counts[i] = counts[i] + 1;
                found_bin = true;
            }
        } else
        {
            if (value <= classes[0])
            {
                counts[i] = counts[i] + 1;
                found_bin = true;
            }
        }
    }

    // not found? add to the last one

    if (!found_bin)
    {
        // add the last class with value 0 if required

        if (counts.size() == classes.size())
        {
            counts.push_back(0);
            cerr << "WARNING: value bigger than highest histogram class. Adding an 'over' bucket for values higher than " << classes[classes.size() - 1] << endl;
        }

        // count up the 'over' bucket

        counts[classes.size()] = counts[classes.size()] + 1;

        exceeded_max_class = true;
    }
}

/** Histogram output */

template <typename T>
void print_histogram(const vector<T> &classes,
        const bin_t &values,
        ofstream &file)
{
    for (size_t i = 0; i < classes.size(); i++)
    {
        file << classes[i] << "," << values[i] << endl;
        cout << classes[i] << "," << values[i] << endl;
    }

    // 'Over' bucket?

    if (values.size() > classes.size())
    {
        file << " > " << classes[classes.size() - 1] << "," << values.back() << endl;
        cout << " > " << classes[classes.size() - 1] << "," << values.back() << endl;
    }
}

template <typename T>
void write_histogram(const std::string &filename,
        const std::string &x,
        const std::string &y,
        const vector<T> &classes,
        const bin_t &values)
{
    ofstream file(filename.c_str());

    file << "#" << x << "  " << y << endl;

    for (size_t i = 0; i < classes.size(); i++)
    {
        file << classes[i] << "  " << values[i] << endl;
    }

    file.close();
}

template <typename T>
void write_values(const std::string &filename,
        const std::string &x,
        const std::string &y,
        const map<T, size_t> &values)
{
    ofstream file(filename.c_str());

    file << "track length  number of tracks" << endl;

    typename map<T, size_t>::const_iterator si;

    for (si = values.begin(); si != values.end(); si++)
    {
        file << si->first << "  " << si->second << endl;
    }

    file.close();
}

template <typename T>
double average(const vector<FS_TYPE> &v)
{
    // calculate the mean value
    double sum = 0.0;
    for (size_t i = 0; i < v.size(); i++)
    {
        sum += boost::numeric_cast<double>(v[i]);
    }
    return sum / boost::numeric_cast<double>(v.size());
}

/** Takes the given 'distribution' and calculates the weighed average
 * @param distribution map
 * @param If <code>true</code> then the class '1' is ignored. 
 *        If <code>false</code> it counts normally.
 */
template <typename T>
double average(const map<T, size_t> &m, bool ignore_one = true)
{
    // calculate the mean value
    double sum = 0.0;

    size_t total_count = 0;

    typename std::map<T, size_t>::const_iterator mi;

    for (mi = m.begin(); mi != m.end(); mi++)
    {
        T key = mi->first;

        size_t val = mi->second;

        if (key == 1 && ignore_one)
        {
            continue;
        }

        total_count += val;

        sum += boost::numeric_cast<double>(mi->first * mi->second);
    }

    return sum / boost::numeric_cast<double>(total_count);
}

#pragma mark -
#pragma mark Memory management

#if APPLE 
#include <mach/vm_statistics.h>
#include <mach/mach_types.h> 
#include <mach/mach_init.h>
#include <mach/mach_host.h>

/** Calculates available and used memory in megabytes
 * 
 * @param free_memory
 * @param used_memory
 */
void get_memory(double &free_memory, double &used_memory)
{
    const double MB = 1024.0 * 1024.0;
    
    vm_size_t page_size;
    mach_port_t mach_port;
    mach_msg_type_number_t count;
    vm_statistics_data_t vm_stats;

    mach_port = mach_host_self();
    
    count = sizeof(vm_stats) / sizeof(natural_t);
    
    if (KERN_SUCCESS == host_page_size(mach_port, &page_size) &&
        KERN_SUCCESS == host_statistics(mach_port, HOST_VM_INFO, 
                                        (host_info_t)&vm_stats, &count))
    {
        free_memory = ((double)vm_stats.free_count * (int64_t)page_size) / MB;
        
        

        used_memory = (((double)vm_stats.active_count + 
                       (double)vm_stats.inactive_count + 
                       (double)vm_stats.wire_count) *  (double)page_size) / MB;
    }
}
#endif

#pragma mark -
#pragma mark main

/**
 *
 *
 */
int main(int argc, char** argv)
{
    using namespace m3D;
    using namespace m3D::utils::vectors;

    size_t length_hist_default_values[] = {5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100, 105, 110};
    bin_t length_hist_default(length_hist_default_values, length_hist_default_values + 22);

    size_t size_hist_default_values[] = {100, 200, 300, 400, 500, 1000, 2000, 3000, 4000, 5000, 7500, 10000, 15000, 20000, 25000, 50000, 100000};
    bin_t size_hist_default(size_hist_default_values, size_hist_default_values + 17);

    size_t cluster_hist_default_values[] = {10, 50, 100, 500, 1000, 5000, 10000, 50000, 100000, 500000, 10000000};
    bin_t cluster_hist_default(cluster_hist_default_values, cluster_hist_default_values + 11);

    float speed_hist_default_values[] = {2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60};
    fvec_t speed_hist_default(speed_hist_default_values, speed_hist_default_values + 30);

    float direction_hist_default_values[] = {15, 30, 45, 60, 75, 90, 105, 120, 135, 150, 175, 200, 215, 230, 245, 260, 275, 300, 315, 330, 345, 360};
    fvec_t direction_hist_default(direction_hist_default_values, direction_hist_default_values + 22);

    // Declare the supported options.

    program_options::options_description desc("Options");
    desc.add_options()
            ("help,h", "produce help message")
            ("version", "print version information and exit")
            ("basename,b", program_options::value<string>()->implicit_value(""), "Basename for filtering input files. Only files starting with this are used. It is also prepended to result files (optional)")
            ("sourcepath,s", program_options::value<string>()->default_value("."), "Current cluster file (netCDF)")
            ("exclude-degenerates,d", program_options::value<bool>()->default_value(true), "Exclude results of tracks of length one")
            ("create-length-statistics,l", "Create a statistic of track lengths.")
            ("length-histogram-classes", program_options::value<bin_t>()->multitoken()->default_value(length_hist_default), "List of track-length values for histogram bins")
            ("create-speed-statistics,v", "Evaluate speeds of clusters in tracks, based on geometric center point")
            ("speed-histogram-classes", program_options::value<fvec_t>()->multitoken()->default_value(speed_hist_default), "Speed histogram. Values in [m/s]")
            ("create-direction-statistics,d", "Evaluate directions of clusters in tracks, based on geometric center point")
            ("direction-histogram-classes", program_options::value<fvec_t>()->multitoken()->default_value(direction_hist_default), "Direction histogram. Values in [deg]. Use with radolan grid only!!")
            ("create-cluster-statistics,c", "Evaluate each cluster in each track in terms of size.")
            ("cluster-histogram-classes", program_options::value<bin_t>()->multitoken()->default_value(cluster_hist_default), "List of cluster size values for histogram bins")
            ("create-cumulated-size-statistics,u", "Evaluate each track in terms of cumulative size. Warning: this process takes a lot of memory.")
            ("size-histogram-classes", program_options::value<bin_t>()->multitoken()->default_value(size_hist_default), "List of cumulated track size values for histogram bins")
            ("write-track-dictionary,t", "Write out a dictionary listing tracks with number of clusters etc.")
#if WITH_VTK
            ("write-center-tracks-as-vtk,e", "Write tracks out as .vtk files")
            ("write-cumulated-tracks-as-vtk,m", "Write cumulated tracks out as .vtk files. Only has effect if --create-cumulated-size-statistics is used")
            ("vtk-dimensions", program_options::value<svec_t>()->multitoken(), "VTK files are written in the order of dimensions given. This may lead to wrong results if the order of the dimensions is not x,y,z. Add the comma-separated list of dimensions here, in the order you would like them to be written as (x,y,z)")
#endif    
            ("write-gnuplot-files,g", "write individual files for the statistics fit for use with gnuplot")
            ;

    program_options::variables_map vm;

    try
    {
        program_options::store(program_options::parse_command_line(argc, argv, desc), vm);
        program_options::notify(vm);
    } catch (std::exception &e)
    {
        cerr << "FATAL:Error parsing command line: " << e.what() << endl;
        exit(EXIT_FAILURE);
    }

    if (vm.count("help") == 1 || argc < 2)
    {
        cout << desc << "\n";
        return EXIT_SUCCESS;
    }

    if (vm.count("version") != 0)
    {
        cout << m3D::VERSION << endl;
        return EXIT_SUCCESS;
{'fraction_non_alphanumeric': 0.1361329900030294, 'fraction_numerical': 0.026810057558315663, 'mean_word_length': 3.034372135655362, 'pattern_counts': {'":': 0, '<': 337, '<?xml version=': 0, '>': 221, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1444763', 'n_tokens_mistral': 9269, 'n_tokens_neox': 8367, 'n_words': 2136}