<commit_before>#include "geopal_parser.h"
#include <boost/graph/strong_components.hpp>
#include <boost/graph/connected_components.hpp>

namespace ed { namespace connectors {

GeopalParser::GeopalParser(const std::string& path, const ed::connectors::ConvCoord& conv_coord): path(path),
                            conv_coord(conv_coord), ways_fusionned(0){
    logger = log4cplus::Logger::getInstance("log");
    try{
            boost::filesystem::path directory(this->path);
            boost::filesystem::directory_iterator iter(directory), end;
             for(;iter != end; ++iter){
                 if (iter->path().extension() == ".txt"){
                    std::string file_name = iter->path().filename().string();
                    this->files.push_back(file_name);
                 }
             }
        }catch( const boost::filesystem::filesystem_error& e){
            throw GeopalParserException("GeopalParser : Error " + e.code().message());
        }
}

bool GeopalParser::starts_with(std::string filename, const std::string& prefex){
    return boost::algorithm::starts_with(filename, prefex);
}





//we affect the first way for every edges of a component
void GeopalParser::fusion_ways_list(const std::vector<ed::types::Edge*>& edges){
    auto begin = edges.begin();
    ed::types::Way* way_ref = (*begin)->way;
    this->data.fusion_ways[way_ref->uri] = way_ref;
    ++begin;
    for(auto it=begin; it != edges.end(); ++it) {
        ed::types::Way* way = (*it)->way;
        way->is_used = false;
        this->data.fusion_ways[way->uri] = way_ref;
        (*it)->way = way_ref;
        this->ways_fusionned++;
    }
}


//We make a graph out of each edge of every edges
//We look for connected components in this graph
void GeopalParser::fusion_ways_by_graph(const std::vector<types::Edge*>& edges){
    typedef boost::adjacency_list<boost::vecS, boost::vecS, boost::directedS, size_t, ed::types::Edge*> Graph;
    Graph graph;
    std::unordered_map<uint64_t, idx_t> node_map_temp;
    // Graph building
    for(ed::types::Edge* edge : edges){
        if(node_map_temp.find(edge->source->id) == node_map_temp.end()){
            node_map_temp[edge->source->id] = boost::add_vertex(graph);
        }
        if(node_map_temp.find(edge->target->id) == node_map_temp.end()){
            node_map_temp[edge->target->id] = boost::add_vertex(graph);
        }
        boost::add_edge(node_map_temp[edge->source->id], node_map_temp[edge->target->id], edge, graph);
        boost::add_edge(node_map_temp[edge->target->id], node_map_temp[edge->source->id], edge, graph);
    }
    std::vector<size_t> vertex_component(boost::num_vertices(graph));
    size_t num = boost::connected_components(graph, &vertex_component[0]);
    if(num != edges.size()){
        std::map<size_t, std::vector<ed::types::Edge*>> component_edges;
        for(size_t i = 0; i<vertex_component.size(); ++i) {
            auto component = vertex_component[i];
            if(component_edges.find(component) != component_edges.end()) {
                component_edges[component] = std::vector<ed::types::Edge*>();
            }
            auto begin_end = boost::out_edges(i, graph);
            for(auto it = begin_end.first; it != begin_end.second; ++it) {
                component_edges[component].push_back(graph[*it]);
            }
        }
        for(auto component_edges_ : component_edges) {
            this->fusion_ways_list(component_edges_.second);
        }
    }
}

// We group the way with the same name and the same admin
void GeopalParser::fusion_ways(){
    typedef std::unordered_map<std::string, std::vector<types::Edge*>> wayname_ways;
    std::unordered_map<std::string, wayname_ways> admin_wayname_way;
    for(auto way : this->data.ways) {
        if(way.second->admin == nullptr || way.second->edges.empty()) {
            continue;
        }
        auto admin_it = admin_wayname_way.find(way.second->admin->insee);
        if(admin_it == admin_wayname_way.end()) {
            admin_wayname_way[way.second->admin->insee] = wayname_ways();
            admin_it = admin_wayname_way.find(way.second->admin->insee);
        }
        auto way_it = admin_it->second.find(way.second->name);
        if(way_it == admin_it->second.end()) {
            admin_wayname_way[way.second->admin->insee][way.second->name] = std::vector<types::Edge*>();
            way_it = admin_it->second.find(way.second->name);
        }
        auto &way_vector = admin_wayname_way[way.second->admin->insee][way.second->name];
        way_vector.insert(way_vector.begin(), way.second->edges.begin(),  way.second->edges.end());
    }
    for(auto wayname_ways_it : admin_wayname_way){
        for(auto edges_it : wayname_ways_it.second){
            this->fusion_ways_list(edges_it.second);
        }
    }
}
<commit_msg>Ed: We ignore way without a name while merging edges<commit_after>#include "geopal_parser.h"
#include <boost/graph/strong_components.hpp>
#include <boost/graph/connected_components.hpp>

namespace ed { namespace connectors {

GeopalParser::GeopalParser(const std::string& path, const ed::connectors::ConvCoord& conv_coord): path(path),
                            conv_coord(conv_coord), ways_fusionned(0){
    logger = log4cplus::Logger::getInstance("log");
    try{
            boost::filesystem::path directory(this->path);
            boost::filesystem::directory_iterator iter(directory), end;
             for(;iter != end; ++iter){
                 if (iter->path().extension() == ".txt"){
                    std::string file_name = iter->path().filename().string();
                    this->files.push_back(file_name);
                 }
             }
        }catch( const boost::filesystem::filesystem_error& e){
            throw GeopalParserException("GeopalParser : Error " + e.code().message());
        }
}

bool GeopalParser::starts_with(std::string filename, const std::string& prefex){
    return boost::algorithm::starts_with(filename, prefex);
}





//we affect the first way for every edges of a component
void GeopalParser::fusion_ways_list(const std::vector<ed::types::Edge*>& edges){
    auto begin = edges.begin();
    ed::types::Way* way_ref = (*begin)->way;
    this->data.fusion_ways[way_ref->uri] = way_ref;
    ++begin;
    for(auto it=begin; it != edges.end(); ++it) {
        ed::types::Way* way = (*it)->way;
        way->is_used = false;
        this->data.fusion_ways[way->uri] = way_ref;
        (*it)->way = way_ref;
        this->ways_fusionned++;
    }
}


//We make a graph out of each edge of every edges
//We look for connected components in this graph
void GeopalParser::fusion_ways_by_graph(const std::vector<types::Edge*>& edges){
    typedef boost::adjacency_list<boost::vecS, boost::vecS, boost::directedS, size_t, ed::types::Edge*> Graph;
    Graph graph;
    std::unordered_map<uint64_t, idx_t> node_map_temp;
    // Graph building
    for(ed::types::Edge* edge : edges){
        if(node_map_temp.find(edge->source->id) == node_map_temp.end()){
            node_map_temp[edge->source->id] = boost::add_vertex(graph);
        }
        if(node_map_temp.find(edge->target->id) == node_map_temp.end()){
            node_map_temp[edge->target->id] = boost::add_vertex(graph);
        }
        boost::add_edge(node_map_temp[edge->source->id], node_map_temp[edge->target->id], edge, graph);
        boost::add_edge(node_map_temp[edge->target->id], node_map_temp[edge->source->id], edge, graph);
    }
    std::vector<size_t> vertex_component(boost::num_vertices(graph));
    size_t num = boost::connected_components(graph, &vertex_component[0]);
    if(num != edges.size()){
        std::map<size_t, std::vector<ed::types::Edge*>> component_edges;
        for(size_t i = 0; i<vertex_component.size(); ++i) {
            auto component = vertex_component[i];
            if(component_edges.find(component) != component_edges.end()) {
                component_edges[component] = std::vector<ed::types::Edge*>();
            }
            auto begin_end = boost::out_edges(i, graph);
            for(auto it = begin_end.first; it != begin_end.second; ++it) {
                component_edges[component].push_back(graph[*it]);
            }
        }
        for(auto component_edges_ : component_edges) {
            this->fusion_ways_list(component_edges_.second);
        }
    }
}

// We group the way with the same name and the same admin
void GeopalParser::fusion_ways(){
    typedef std::unordered_map<std::string, std::vector<types::Edge*>> wayname_ways;
    std::unordered_map<std::string, wayname_ways> admin_wayname_way;
    for(auto way : this->data.ways) {
        if(way.second->admin == nullptr || way.second->edges.empty() || way.second->name == "") {
            continue;
        }
        auto admin_it = admin_wayname_way.find(way.second->admin->insee);
        if(admin_it == admin_wayname_way.end()) {
            admin_wayname_way[way.second->admin->insee] = wayname_ways();
            admin_it = admin_wayname_way.find(way.second->admin->insee);
        }
        auto way_it = admin_it->second.find(way.second->name);
        if(way_it == admin_it->second.end()) {
            admin_wayname_way[way.second->admin->insee][way.second->name] = std::vector<types::Edge*>();
            way_it = admin_it->second.find(way.second->name);
        }
        auto &way_vector = admin_wayname_way[way.second->admin->insee][way.second->name];
        way_vector.insert(way_vector.begin(), way.second->edges.begin(),  way.second->edges.end());
    }
    for(auto wayname_ways_it : admin_wayname_way){
        for(auto edges_it : wayname_ways_it.second){
            this->fusion_ways_list(edges_it.second);
        }
    }
}

{'fraction_non_alphanumeric': 0.15482939903102774, 'fraction_numerical': 0.0012369858777445625, 'mean_word_length': 3.1040609137055837, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 138, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1352626', 'n_tokens_mistral': 3092, 'n_tokens_neox': 3022, 'n_words': 502}