<commit_before>#include "cut.hpp"

/*

Hardcut Algorithm
 - walk over all node-versions
   - if the current node-version has a new node-id
     - walk over all bboxes
       - if node-writing for this bbox is enabled
         - write all nodes from the current-node-vector to this bboxes writer
      - disable node-writing for this bbox
      - clear the current-node-vector

   - add the node-version to the current-node-vector

   - walk over all bboxes
     - if node-writing for this bbox is still disabled
       - if the node-version is in the bbox
         - enable node-writing for this bbox
         - record its id in the bboxes node-id-tracker



 - walk over all way-versions
   - if the current way-version has a new way-id
     - walk over all bboxes
       - if way-writing for this bbox is enabled
         - write all ways from the current-way-vector to this bboxes writer
      - disable way-writing for this bbox
      - clear the current-way-vector

   - walk over all bboxes
     - create a new way NULL pointer
     - walk over all waynodes
       - if the waynode is in the node-id-tracker of this bbox
         - if the new way pointer is NULL
           - create a new way with all meta-data and tags but without waynodes
         - add the waynode to the new way

     - if the way pointer is not NULL
       - enable way-writing for this bbox
       - add the way to the current-way-vector
       - record its id in the bboxes way-id-tracker



 - walk over all relation-versions
   - if the current relation-version has a new relation-id
     - walk over all bboxes
       - if relation-writing for this bbox is enabled
         - write all relations from the current-relation-vector to this bboxes writer
      - disable relation-writing for this bbox
      - clear the current-relation-vector

   - add the relation-version to the current-relation-vector

   - walk over all bboxes
     - create a new relation NULL pointer
     - walk over all relation members
       - if the relation members is in the node-id-tracker or the way-id-tracker of this bbox
         - if the new relation pointer is NULL
           - create a new relation with all meta-data and tags but without members
         - add the member to the new relation

     - if the relation pointer is not NULL
       - enable relation-writing for this bbox
       - add the relation to the current-relations-vector

advantages:
 - single pass
 - all versions of elements are reserved
 - ways and relations are cropped at bbox boundaries

Disadvantages
 - Relations referring to Relations that come later in the file are missing this references

*/

class Hardcut : public Cut {

protected:

    std::vector<Osmium::OSM::Node*> current_node_vector;

    osm_object_id_t last_id;

public:

    void callback_init() {
        printf("hardcut init\n");
        for(int i = 0, l = bboxes.size(); i<l; i++) {
            printf("\tbbox[%d] %s\n", i, bboxes[i]->name.c_str());
        }

        last_id = 0;
        
        if(debug) fprintf(stderr, "\n\n===== NODES =====\n\n");
    }

    // walk over all node-versions
    void callback_node(Osmium::OSM::Node *e) {
        if(debug) printf("hardcut node %d v%d\n", e->id, e->version);

        // if the current node-version has a new node-id
        if(last_id > 0 && last_id != e->id) {
            // post-process this node
            if(debug) fprintf(stderr, "new node-id %d (last one was %d)\n", e->id, last_id);
            post_node_proc();
        }

        // add the node-version to the current-node-vector
        if(debug) fprintf(stderr, "pushing node %d v%d into current_node_vector\n", e->id, e->version);
        Osmium::OSM::Node *clone = new Osmium::OSM::Node(*e);
        current_node_vector.push_back(clone);

        // walk over all bboxes
        for(int i = 0, l = bboxes.size(); i<l; i++) {
            // shorthand
            BBoxInfo *bbox = bboxes[i];

            // if node-writing for this bbox is still disabled
            if(!bbox->enabled) {

                // if the node-version is in the bbox
                if(debug) fprintf(stderr, "bbox[%d]-check lat(%f < %f < %f) && lon(%f < %f < %f)\n", i, bbox->x1, e->get_lat(), bbox->x2, bbox->y1, e->get_lon(), bbox->y2);
                if(e->get_lat() > bboxes[i]->x1 && e->get_lat() < bbox->x2 && e->get_lon() > bbox->y1 && e->get_lon() < bbox->y2) {

                    if(debug) fprintf(stderr, "node %d v%d is inside bbox[%d], enabling node-writing for bbox[%d]\n", e->id, e->version, i, i);

                    // enable node-writing for this bbox
                    bbox->enabled = true;

                    // record its id in the bboxes node-id-tracker
                    if((int)bbox->node_tracker.size() < e->id) {
                        fprintf(stderr, "WARNING! node_tracker is too small to hold id %d, resizing...\n", e->id);
                        fprintf(stderr, "    TIP: increase estimation of max. node id in cut.hpp\n");
                        bbox->node_tracker.reserve(e->id);
                    }
                    bbox->node_tracker[e->id] = true;
                }
            }
        }

        // record the last id
        last_id = e->id;
    }
    
    void post_node_proc() {
        if(debug) fprintf(stderr, "doing post-node processing\n");
        
        // walk over all bboxes
        for(int i = 0, l = bboxes.size(); i<l; i++) {
            // shorthand
            BBoxInfo *bbox = bboxes[i];

            // if node-writing for this bbox is enabled
            if(bbox->enabled) {

                if(debug) fprintf(stderr, "node-writing is enabled for bbox[%d]\n", i);

                // write all nodes from the current-node-vector to this bboxes writer
                for(int ii = 0, ll = current_node_vector.size(); ii < ll; ii++) {

                    Osmium::OSM::Node *cur = current_node_vector[ii];

                    if(debug) fprintf(stderr, "writing node %d v%d (index %d in current_node_vector) to writer of bbox[%d]\n", cur->id, cur->version, ii, i);

                    // bboxes writer
                    bbox->writer->write(cur);
                }

                // disable node-writing for this bbox
                bbox->enabled = false;
            }
         }

         // clear the current-node-vector
         if(debug) fprintf(stderr, "clearing current_node_vector\n");
         for(int ii = 0, ll = current_node_vector.size(); ii < ll; ii++) {
            delete current_node_vector[ii];
         }
         current_node_vector.clear();
    }

    void callback_after_nodes() {
        if(debug) fprintf(stderr, "after nodes\n");
        post_node_proc();
        last_id = 0;
        
        if(debug) fprintf(stderr, "\n\n===== WAYS =====\n\n");
    }

    // walk over all way-versions
    void callback_way(Osmium::OSM::Way *e) {
        if(debug) printf("hardcut way %d v%d\n", e->id, e->version);

        // if the current way-version has a new way-id
        if(last_id > 0 && last_id != e->id) {
            // post-process this way
            if(debug) fprintf(stderr, "new way-id %d (last one was %d)\n", e->id, last_id);
            post_way_proc();
        }

        // walk over all bboxes
        for(int i = 0, l = bboxes.size(); i<l; i++) {
            // shorthand
            BBoxInfo *bbox = bboxes[i];

            // create a new way NULL pointer
            Osmium::OSM::Way *c = NULL;

            // walk over all waynodes
            for(int ii = 0, ll = e->node_count(); ii < ll; ii++) {
                // shorthand
                osm_object_id_t node_id = e->nodes[ii];

                // if the waynode is in the node-id-tracker of this bbox
                if(bbox->node_tracker[node_id]) {
                    // if the new way pointer is NULL
                    if(!c) {
                        // create a new way with all meta-data and tags but without waynodes
                        if(debug) fprintf(stderr, "creating cutted way %d v%d for bbox[%d]\n", e->id, e->version, i);
                        c = Osmium::OSM::Way::clone_meta(*e);
                    }

                    // add the waynode to the new way
                    if(debug) fprintf(stderr, "adding node-id %d to cutted way %d v%d for bbox[%d]\n", node_id, e->id, e->version, i);
                    c->add_node(node_id);
                }
            }

            // if the way pointer is not NULL
            if(c) {
                // enable way-writing for this bbox
                if(debug) fprintf(stderr, "way %d v%d is in bbox[%d]\n", e->id, e->version, i);

                // check for short ways
                if(c->node_count() < 2) {
                    if(debug) fprintf(stderr, "way %d v%d in bbox[%d] would only be %d nodes long, skipping\n", e->id, e->version, i, c->node_count());
                    delete c;
                    c = NULL;
                    continue;
                }

                bbox->enabled = true;

                // add the way to the current-way-vector
                if(debug) fprintf(stderr, "pushing way %d v%d into current_way_vector of bbox[%d]\n", e->id, e->version, i);
                bbox->way_vector.push_back(c);

                // record its id in the bboxes way-id-tracker
                if((int)bbox->way_tracker.size() < e->id) {
                    fprintf(stderr, "WARNING! way_tracker is too small to hold id %d, resizing...\n", e->id);
                    fprintf(stderr, "    TIP: increase estimation of max. way id in cut.hpp\n");
                    bbox->node_tracker.reserve(e->id);
                }
                bbox->way_tracker[e->id] = true;
            }
        }

        // record the last id
        last_id = e->id;
    }

    void post_way_proc() {
        if(debug) fprintf(stderr, "doing post-way processing\n");

        // walk over all bboxes
        for(int i = 0, l = bboxes.size(); i<l; i++) {
            // shorthand
            BBoxInfo *bbox = bboxes[i];

            // if way-writing for this bbox is enabled
            if(bbox->enabled) {

                if(debug) fprintf(stderr, "way-writing is enabled for bbox[%d]\n", i);

                // write all ways from the current-way-vector to this bboxes writer
                for(int ii = 0, ll = bbox->way_vector.size(); ii < ll; ii++) {

                    Osmium::OSM::Way *cur = bbox->way_vector[ii];

                    if(debug) fprintf(stderr, "writing way %d v%d (index %d in current_way_vector) to writer of bbox[%d]\n", cur->id, cur->version, ii, i);

                    // bboxes writer
                    bbox->writer->write(cur);
                }

                // disable way-writing for this bbox
                bbox->enabled = false;
            }

            // clear the current-way-vector
            if(debug) fprintf(stderr, "clearing current_way_vector of bbox[%d]\n", i);
            for(int ii = 0, ll = bbox->way_vector.size(); ii < ll; ii++) {
                delete bbox->way_vector[ii];
            }
            bbox->way_vector.clear();
         }
    }

    void callback_after_ways() {
        if(debug) fprintf(stderr, "after ways\n");
        post_way_proc();
        last_id = 0;

        if(debug) fprintf(stderr, "\n\n===== RELATIONS =====\n\n");
    }

    void callback_relation(Osmium::OSM::Relation *e) {
        //printf("hardcut relation\n");
        //for(int i = 0, l = bboxes.size(); i<l; i++)
        //    bboxes[i].writer->write(e);
    }

    void callback_final() {
        printf("hardcut finished\n");
    }
};

<commit_msg>algorithm comments<commit_after>#include "cut.hpp"

/*

Hardcut Algorithm
 - walk over all node-versions
   - if the current node-version has a new node-id
     - walk over all bboxes
       - if node-writing for this bbox is enabled
         - write all nodes from the current-node-vector to this bboxes writer
      - disable node-writing for this bbox
      - clear the current-node-vector

   - add the node-version to the current-node-vector

   - walk over all bboxes
     - if node-writing for this bbox is still disabled
       - if the node-version is in the bbox
         - enable node-writing for this bbox
         - record its id in the bboxes node-id-tracker



 - walk over all way-versions
   - if the current way-version has a new way-id
     - walk over all bboxes
       - if way-writing for this bbox is enabled
         - write all ways from the current-way-vector to this bboxes writer
      - disable way-writing for this bbox
      - clear the current-way-vector

   - walk over all bboxes
     - create a new way NULL pointer
     - walk over all waynodes
       - if the waynode is in the node-id-tracker of this bbox
         - if the new way pointer is NULL
           - create a new way with all meta-data and tags but without waynodes
         - add the waynode to the new way

     - if the way pointer is not NULL
       - if the way has <2 waynodes
         - delete it and continue with the next way-version
       - enable way-writing for this bbox
       - add the way to the current-way-vector
       - record its id in the bboxes way-id-tracker



 - walk over all relation-versions
   - if the current relation-version has a new relation-id
     - walk over all bboxes
       - if relation-writing for this bbox is enabled
         - write all relations from the current-relation-vector to this bboxes writer
      - disable relation-writing for this bbox
      - clear the current-relation-vector

   - walk over all bboxes
     - create a new relation NULL pointer
     - walk over all relation members
       - if the relation members is in the node-id-tracker or the way-id-tracker of this bbox
         - if the new relation pointer is NULL
           - create a new relation with all meta-data and tags but without members
         - add the member to the new relation

     - if the relation pointer is not NULL
       - enable relation-writing for this bbox
       - add the relation to the current-relations-vector

advantages:
 - single pass
 - all versions of elements are reserved
 - ways and relations are cropped at bbox boundaries

Disadvantages
 - relations referring to relations that come later in the file are missing this references
 - ways that have only one node in the bbox are missing from the output
 - implementation is complex and error prone

*/

class Hardcut : public Cut {

protected:

    std::vector<Osmium::OSM::Node*> current_node_vector;

    osm_object_id_t last_id;

public:

    void callback_init() {
        printf("hardcut init\n");
        for(int i = 0, l = bboxes.size(); i<l; i++) {
            printf("\tbbox[%d] %s\n", i, bboxes[i]->name.c_str());
        }

        last_id = 0;
        
        if(debug) fprintf(stderr, "\n\n===== NODES =====\n\n");
    }

    // walk over all node-versions
    void callback_node(Osmium::OSM::Node *e) {
        if(debug) printf("hardcut node %d v%d\n", e->id, e->version);

        // if the current node-version has a new node-id
        if(last_id > 0 && last_id != e->id) {
            // post-process this node
            if(debug) fprintf(stderr, "new node-id %d (last one was %d)\n", e->id, last_id);
            post_node_proc();
        }

        // add the node-version to the current-node-vector
        if(debug) fprintf(stderr, "pushing node %d v%d into current_node_vector\n", e->id, e->version);
        Osmium::OSM::Node *clone = new Osmium::OSM::Node(*e);
        current_node_vector.push_back(clone);

        // walk over all bboxes
        for(int i = 0, l = bboxes.size(); i<l; i++) {
            // shorthand
            BBoxInfo *bbox = bboxes[i];

            // if node-writing for this bbox is still disabled
            if(!bbox->enabled) {

                // if the node-version is in the bbox
                if(debug) fprintf(stderr, "bbox[%d]-check lat(%f < %f < %f) && lon(%f < %f < %f)\n", i, bbox->x1, e->get_lat(), bbox->x2, bbox->y1, e->get_lon(), bbox->y2);
                if(e->get_lat() > bboxes[i]->x1 && e->get_lat() < bbox->x2 && e->get_lon() > bbox->y1 && e->get_lon() < bbox->y2) {

                    if(debug) fprintf(stderr, "node %d v%d is inside bbox[%d], enabling node-writing for bbox[%d]\n", e->id, e->version, i, i);

                    // enable node-writing for this bbox
                    bbox->enabled = true;

                    // record its id in the bboxes node-id-tracker
                    if((int)bbox->node_tracker.size() < e->id) {
                        fprintf(stderr, "WARNING! node_tracker is too small to hold id %d, resizing...\n", e->id);
                        fprintf(stderr, "    TIP: increase estimation of max. node id in cut.hpp\n");
                        bbox->node_tracker.reserve(e->id);
                    }
                    bbox->node_tracker[e->id] = true;
                }
            }
        }

        // record the last id
        last_id = e->id;
    }
    
    void post_node_proc() {
        if(debug) fprintf(stderr, "doing post-node processing\n");
        
        // walk over all bboxes
        for(int i = 0, l = bboxes.size(); i<l; i++) {
            // shorthand
            BBoxInfo *bbox = bboxes[i];

            // if node-writing for this bbox is enabled
            if(bbox->enabled) {

                if(debug) fprintf(stderr, "node-writing is enabled for bbox[%d]\n", i);

                // write all nodes from the current-node-vector to this bboxes writer
                for(int ii = 0, ll = current_node_vector.size(); ii < ll; ii++) {

                    Osmium::OSM::Node *cur = current_node_vector[ii];

                    if(debug) fprintf(stderr, "writing node %d v%d (index %d in current_node_vector) to writer of bbox[%d]\n", cur->id, cur->version, ii, i);

                    // bboxes writer
                    bbox->writer->write(cur);
                }

                // disable node-writing for this bbox
                bbox->enabled = false;
            }
         }

         // clear the current-node-vector
         if(debug) fprintf(stderr, "clearing current_node_vector\n");
         for(int ii = 0, ll = current_node_vector.size(); ii < ll; ii++) {
            delete current_node_vector[ii];
         }
         current_node_vector.clear();
    }

    void callback_after_nodes() {
        if(debug) fprintf(stderr, "after nodes\n");
        post_node_proc();
        last_id = 0;
        
        if(debug) fprintf(stderr, "\n\n===== WAYS =====\n\n");
    }

    // walk over all way-versions
    void callback_way(Osmium::OSM::Way *e) {
        if(debug) printf("hardcut way %d v%d\n", e->id, e->version);

        // if the current way-version has a new way-id
        if(last_id > 0 && last_id != e->id) {
            // post-process this way
            if(debug) fprintf(stderr, "new way-id %d (last one was %d)\n", e->id, last_id);
            post_way_proc();
        }

        // walk over all bboxes
        for(int i = 0, l = bboxes.size(); i<l; i++) {
            // shorthand
            BBoxInfo *bbox = bboxes[i];

            // create a new way NULL pointer
            Osmium::OSM::Way *c = NULL;

            // walk over all waynodes
            for(int ii = 0, ll = e->node_count(); ii < ll; ii++) {
                // shorthand
                osm_object_id_t node_id = e->nodes[ii];

                // if the waynode is in the node-id-tracker of this bbox
                if(bbox->node_tracker[node_id]) {
                    // if the new way pointer is NULL
                    if(!c) {
                        // create a new way with all meta-data and tags but without waynodes
                        if(debug) fprintf(stderr, "creating cutted way %d v%d for bbox[%d]\n", e->id, e->version, i);
                        c = Osmium::OSM::Way::clone_meta(*e);
                    }

                    // add the waynode to the new way
                    if(debug) fprintf(stderr, "adding node-id %d to cutted way %d v%d for bbox[%d]\n", node_id, e->id, e->version, i);
                    c->add_node(node_id);
                }
            }

            // if the way pointer is not NULL
            if(c) {
                // enable way-writing for this bbox
                if(debug) fprintf(stderr, "way %d v%d is in bbox[%d]\n", e->id, e->version, i);

                // check for short ways
                if(c->node_count() < 2) {
                    if(debug) fprintf(stderr, "way %d v%d in bbox[%d] would only be %d nodes long, skipping\n", e->id, e->version, i, c->node_count());
                    delete c;
                    c = NULL;
                    continue;
                }

                bbox->enabled = true;

                // add the way to the current-way-vector
                if(debug) fprintf(stderr, "pushing way %d v%d into current_way_vector of bbox[%d]\n", e->id, e->version, i);
                bbox->way_vector.push_back(c);

                // record its id in the bboxes way-id-tracker
                if((int)bbox->way_tracker.size() < e->id) {
                    fprintf(stderr, "WARNING! way_tracker is too small to hold id %d, resizing...\n", e->id);
                    fprintf(stderr, "    TIP: increase estimation of max. way id in cut.hpp\n");
                    bbox->node_tracker.reserve(e->id);
                }
                bbox->way_tracker[e->id] = true;
            }
        }

        // record the last id
        last_id = e->id;
    }

    void post_way_proc() {
        if(debug) fprintf(stderr, "doing post-way processing\n");

        // walk over all bboxes
        for(int i = 0, l = bboxes.size(); i<l; i++) {
            // shorthand
            BBoxInfo *bbox = bboxes[i];

            // if way-writing for this bbox is enabled
            if(bbox->enabled) {

                if(debug) fprintf(stderr, "way-writing is enabled for bbox[%d]\n", i);

                // write all ways from the current-way-vector to this bboxes writer
                for(int ii = 0, ll = bbox->way_vector.size(); ii < ll; ii++) {

                    Osmium::OSM::Way *cur = bbox->way_vector[ii];

                    if(debug) fprintf(stderr, "writing way %d v%d (index %d in current_way_vector) to writer of bbox[%d]\n", cur->id, cur->version, ii, i);

                    // bboxes writer
                    bbox->writer->write(cur);
                }

                // disable way-writing for this bbox
                bbox->enabled = false;
            }

            // clear the current-way-vector
            if(debug) fprintf(stderr, "clearing current_way_vector of bbox[%d]\n", i);
            for(int ii = 0, ll = bbox->way_vector.size(); ii < ll; ii++) {
                delete bbox->way_vector[ii];
            }
            bbox->way_vector.clear();
         }
    }

    void callback_after_ways() {
        if(debug) fprintf(stderr, "after ways\n");
        post_way_proc();
        last_id = 0;

        if(debug) fprintf(stderr, "\n\n===== RELATIONS =====\n\n");
    }

    void callback_relation(Osmium::OSM::Relation *e) {
        //printf("hardcut relation\n");
        //for(int i = 0, l = bboxes.size(); i<l; i++)
        //    bboxes[i].writer->write(e);
    }

    void callback_final() {
        printf("hardcut finished\n");
    }
};


{'fraction_non_alphanumeric': 0.12024340075419952, 'fraction_numerical': 0.002185464518340761, 'mean_word_length': 1.65979028949168, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 175, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5467890', 'n_tokens_mistral': 7206, 'n_tokens_neox': 6858, 'n_words': 2199}