<commit_before>#ifndef _M3D_ClusterList_H_
#define _M3D_ClusterList_H_

#include <meanie3D/defines.h>
#include <meanie3D/namespaces.h>

#include <vector>
#include <map>
#include <assert.h>
#include <boost/shared_ptr.hpp>
#include <stdlib.h>
#include <netcdf>

#include <cf-algorithms/cf-algorithms.h>

#include <meanie3D/types.h>

namespace m3D {
    
    using namespace netCDF;
	using namespace std;
    using namespace cfa;
	using ::cfa::meanshift::Point;
    using ::cfa::meanshift::WeightFunction;

    /** Cluster of points in feature space. A cluster is a point in feature space,
     * where multiple trajectories of original feature space points end. This end
     * point is called the cluster's 'mode'.
     *
     * At the same time, this class represents a serialized form of a list of clusters
     * to be used in further processing, such as tracking. Serialization/Deserialization
     * to and from files are done using the read/write methods.
     *
     * TODO: decouple the cluster list from the NetCDF layer
     */
    template <class T>
    class ClusterList
    {
        
    private:
        
        typedef map< vector<T>, typename Cluster<T>::ptr > ClusterMap;
        
        ClusterMap      m_cluster_map;
        
        
        /** Recursive helper method fort find_neighbours
         *
         */
        void
        find_neighbours(typename CoordinateSystem<T>::GridPoint &gridpoint,
                        size_t dim_index,
                        ArrayIndex<T> &index,
                        typename Point<T>::list &list,
                        size_t reach);
        
        typename Point<T>::list
        find_neighbours(const typename CoordinateSystem<T>::GridPoint &gridpoint,
                        ArrayIndex<T> &index,
                        size_t reach=1);
        
        /** Estimate the tendency of the weight function at the given
         * grid point with respect to it's direct (9/16) neighbours.
         * @param point
         * @param weight function
         * @param index
         * @return positive: point is a hole, negative: point is a hill, zero: inconclusive
         */
        T
        weight_function_tendency(typename Point<T>::ptr p,
                                 const WeightFunction<T> *weight_function,
                                 const typename Point<T>::list &neighbours,
                                 ArrayIndex<T> &index);
        
        /** Sanity check / consistency check
         */
        void
        check_clusters(ArrayIndex<T> &index);
        

        void
        aggregate_zeroshifts(FeatureSpace<T> *fs,
                             const WeightFunction<T> *weight_function,
                             ArrayIndex<T> &index,
                             bool coalesceWithStrongestNeighbour,
                             bool show_progress);
        
    public:
        
#pragma mark -
#pragma mark Public typedefs
        
        typedef ClusterList<T> *    ptr;
        
#pragma mark -
#pragma mark Public Properties

        
        NcFile          *ncFile;
        std::string     filename;                       // this is filled on read() or write()
        
        // meta-info
        
        std::vector<NcVar>  feature_variables;    // all variables, including dimension variables
        vector<string>      variable_names;       // vector of strings. this shit needs cleaning up so bad, man
        
        vector<NcDim>       dimensions;           // all dimensions that were in the featurespace
        string              source_file;          // Name of the file that the clusters were created from
        
        // payload

        typename Cluster<T>::list   clusters;
        
        // tracking help
        
        bool            tracking_performed; 
        int             tracking_time_difference; // time difference between the two files in [s]
        id_set_t        tracked_ids;    // IDs that were continues
        id_set_t        dropped_ids;    // IDs that were discontinued
        id_set_t        new_ids;        // IDs that were added fresh
        id_map_t        splits;         // contains a mapping from previous id's to a list of current id's.
        id_map_t        merges;         // contains a mapping of current id to previous ids
        cfa::id_t       highest_id;     // Highest used ID
        
        timestamp_t     timestamp;      // contains the variable 'time(time)' value
        
        // control
        
        bool            m_use_original_points_only;

#pragma mark -
#pragma mark Constructor/Destructor
        
        ClusterList() {};

        /** @constructor
         * @param all variables used to construct the points from, in the same order
         * @param all dimensions that were in the featurespace
         * @param name of the file the clusters were created from
         * @param the command line parameters used to cluster
         * @param when searching for predecessors in the meanshift vector graph,
         *          there is a maximum distance in gridpoints, after which the 
         *          neighbourhood search will not accept points. This parameter
         *          sets that distance in #grid-points.
         * @param In the process of linking up meanshift vectors to a graph, the
         *          vicinity of vector endpoints is searched for other points to
         *          link up to. The maximum distance of that search (in grid points)
         *          is determined here.
         * @param When adding points to clusters, this switch can be used to control
         *          if only points from the original data set are added or also points
         *          that resulted from filtering (for example scale-space)
         */
        ClusterList(const vector<NcVar> &vars,
                    const vector<NcDim> &dims,
                    const string& sourcefile,
                    bool use_original_points_only=true)
        : ncFile(NULL)
        , feature_variables(vars)
        , variable_names(::cfa::utils::netcdf::to_names(vars))
        , dimensions(dims)
        , source_file(sourcefile)
        , tracking_performed(false)
        , highest_id(0)
        , m_use_original_points_only(use_original_points_only)
        {
            long timestamp = cfa::utils::netcdf::get_time<long>(sourcefile);
            this->set_time_in_seconds(::units::values::s(timestamp));
        };

        /** @constructor
         * @param a list of clusters
         * @param all variables used to contruct the points from, in the same order
         * @param the spatial dimension (2D/3D)
         * @param name of the file the clusters were created from
         * @param the command line parameters used to cluster
         */
        ClusterList(const typename Cluster<T>::list &list,
                    const vector<NcDim> &dims,
                    const vector<NcVar> &vars,
                    const string& sourcefile )
        : ncFile(NULL)
        , feature_variables(vars)
        , variable_names(::cfa::utils::netcdf::to_names(vars))
        , dimensions(dims)
        , source_file(sourcefile)
        , clusters(list)
        , tracking_performed(false)
        , highest_id(0)
        , m_use_original_points_only(true)
        {
            for (size_t i=0; i < dims.size(); i++)
                dimensions.push_back(dims[i].getName());
            
            for (size_t i=0; i < vars.size(); i++)
                feature_variables.push_back(vars[i].getName());

            long timestamp = cfa::utils::netcdf::get_time<long> (sourcefile);
            this->set_time_in_seconds(::units::values::s(timestamp));<commit_msg>Fixed a bug in cluster list<commit_after>#ifndef _M3D_ClusterList_H_
#define _M3D_ClusterList_H_

#include <meanie3D/defines.h>
#include <meanie3D/namespaces.h>

#include <vector>
#include <map>
#include <assert.h>
#include <boost/shared_ptr.hpp>
#include <stdlib.h>
#include <netcdf>

#include <cf-algorithms/cf-algorithms.h>

#include <meanie3D/types.h>

namespace m3D {
    
    using namespace netCDF;
	using namespace std;
    using namespace cfa;
	using ::cfa::meanshift::Point;
    using ::cfa::meanshift::WeightFunction;

    /** Cluster of points in feature space. A cluster is a point in feature space,
     * where multiple trajectories of original feature space points end. This end
     * point is called the cluster's 'mode'.
     *
     * At the same time, this class represents a serialized form of a list of clusters
     * to be used in further processing, such as tracking. Serialization/Deserialization
     * to and from files are done using the read/write methods.
     *
     * TODO: decouple the cluster list from the NetCDF layer
     */
    template <class T>
    class ClusterList
    {
        
    private:
        
        typedef map< vector<T>, typename Cluster<T>::ptr > ClusterMap;
        
        ClusterMap      m_cluster_map;
        
        
        /** Recursive helper method fort find_neighbours
         *
         */
        void
        find_neighbours(typename CoordinateSystem<T>::GridPoint &gridpoint,
                        size_t dim_index,
                        ArrayIndex<T> &index,
                        typename Point<T>::list &list,
                        size_t reach);
        
        typename Point<T>::list
        find_neighbours(const typename CoordinateSystem<T>::GridPoint &gridpoint,
                        ArrayIndex<T> &index,
                        size_t reach=1);
        
        /** Estimate the tendency of the weight function at the given
         * grid point with respect to it's direct (9/16) neighbours.
         * @param point
         * @param weight function
         * @param index
         * @return positive: point is a hole, negative: point is a hill, zero: inconclusive
         */
        T
        weight_function_tendency(typename Point<T>::ptr p,
                                 const WeightFunction<T> *weight_function,
                                 const typename Point<T>::list &neighbours,
                                 ArrayIndex<T> &index);
        
        /** Sanity check / consistency check
         */
        void
        check_clusters(ArrayIndex<T> &index);
        

        void
        aggregate_zeroshifts(FeatureSpace<T> *fs,
                             const WeightFunction<T> *weight_function,
                             ArrayIndex<T> &index,
                             bool coalesceWithStrongestNeighbour,
                             bool show_progress);
        
    public:
        
#pragma mark -
#pragma mark Public typedefs
        
        typedef ClusterList<T> *    ptr;
        
#pragma mark -
#pragma mark Public Properties

        
        NcFile          *ncFile;
        std::string     filename;                       // this is filled on read() or write()
        
        // meta-info
        
        std::vector<NcVar>  feature_variables;    // all variables, including dimension variables
        vector<string>      variable_names;       // vector of strings. this shit needs cleaning up so bad, man
        
        vector<NcDim>       dimensions;           // all dimensions that were in the featurespace
        string              source_file;          // Name of the file that the clusters were created from
        
        // payload

        typename Cluster<T>::list   clusters;
        
        // tracking help
        
        bool            tracking_performed; 
        int             tracking_time_difference; // time difference between the two files in [s]
        id_set_t        tracked_ids;    // IDs that were continues
        id_set_t        dropped_ids;    // IDs that were discontinued
        id_set_t        new_ids;        // IDs that were added fresh
        id_map_t        splits;         // contains a mapping from previous id's to a list of current id's.
        id_map_t        merges;         // contains a mapping of current id to previous ids
        cfa::id_t       highest_id;     // Highest used ID
        
        timestamp_t     timestamp;      // contains the variable 'time(time)' value
        
        // control
        
        bool            m_use_original_points_only;

#pragma mark -
#pragma mark Constructor/Destructor
        
        ClusterList() {};

        /** @constructor
         * @param all variables used to construct the points from, in the same order
         * @param all dimensions that were in the featurespace
         * @param name of the file the clusters were created from
         * @param the command line parameters used to cluster
         * @param when searching for predecessors in the meanshift vector graph,
         *          there is a maximum distance in gridpoints, after which the 
         *          neighbourhood search will not accept points. This parameter
         *          sets that distance in #grid-points.
         * @param In the process of linking up meanshift vectors to a graph, the
         *          vicinity of vector endpoints is searched for other points to
         *          link up to. The maximum distance of that search (in grid points)
         *          is determined here.
         * @param When adding points to clusters, this switch can be used to control
         *          if only points from the original data set are added or also points
         *          that resulted from filtering (for example scale-space)
         */
        ClusterList(const vector<NcVar> &vars,
                    const vector<NcDim> &dims,
                    const string& sourcefile,
                    bool use_original_points_only=true)
        : ncFile(NULL)
        , feature_variables(vars)
        , variable_names(::cfa::utils::netcdf::to_names(vars))
        , dimensions(dims)
        , source_file(sourcefile)
        , tracking_performed(false)
        , highest_id(0)
        , m_use_original_points_only(use_original_points_only)
        {
            long timestamp = cfa::utils::netcdf::get_time<long>(sourcefile);
            this->set_time_in_seconds(::units::values::s(timestamp));
        };

        /** @constructor
         * @param a list of clusters
         * @param all variables used to contruct the points from, in the same order
         * @param the spatial dimension (2D/3D)
         * @param name of the file the clusters were created from
         * @param the command line parameters used to cluster
         */
        ClusterList(const typename Cluster<T>::list &list,
                    const vector<NcDim> &dims,
                    const vector<NcVar> &vars,
                    const string& sourcefile )
        : ncFile(NULL)
        , feature_variables(vars)
        , variable_names(::cfa::utils::netcdf::to_names(vars))
        , dimensions(dims)
        , source_file(sourcefile)
        , clusters(list)
        , tracking_performed(false)
        , highest_id(0)
        , m_use_original_points_only(true)
        {
            long timestamp = cfa::utils::netcdf::get_time<long> (sourcefile);
            this->set_time_in_seconds(::units::values::s(timestamp));
{'fraction_non_alphanumeric': 0.07511365882585491, 'fraction_numerical': 0.0019766752322593396, 'mean_word_length': 1.7586332242820792, 'pattern_counts': {'":': 0, '<': 85, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 2, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1441466', 'n_tokens_mistral': 3986, 'n_tokens_neox': 3864, 'n_words': 1248}