<commit_before>/*
    open source routing machine
    Copyright (C) <NAME>, others 2010

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU AFFERO General Public License as published by
the Free Software Foundation; either version 3 of the License, or
any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
or see http://www.gnu.org/licenses/agpl.txt.
 */

#include <cassert>
#include <climits>
#include <cstdlib>
#include <iostream>
#include <string>
#include <sstream>
#include <vector>
#include <libxml/xmlreader.h>

#include "typedefs.h"

using namespace std;

struct _Node : NodeInfo{
    bool trafficSignal:1;
};

struct _Way {
    std::vector< NodeID > path;
    enum {
        notSure = 0, oneway, bidirectional, opposite
    } direction;
    double maximumSpeed;
    bool usefull:1;
    bool access:1;
    short type;
};

typedef google::dense_hash_map<NodeID, _Node> NodeMap;

struct _Stats {
    NodeID numberOfNodes;
    NodeID numberOfEdges;
    NodeID numberOfWays;
    //    NodeID numberOfPlaces;
    //    NodeID numberOfOutlines;
    NodeID numberOfMaxspeed;
    //    NodeID numberOfZeroSpeed;
    //    NodeID numberOfDefaultCitySpeed;
    //    NodeID numberOfCityEdges;
};

struct Settings {
    struct SpeedProfile {
        vector< double > speed;
        vector< string > names;
    } speedProfile;
    vector<string> accessList;
    int trafficLightPenalty;
    int indexInAccessListOf( const string & key)
    {
        for(int i = 0; i< accessList.size(); i++)
        {
            if(accessList[i] == key)
                return i;
        }
        return -1;
    }
};

_Way _ReadXMLWay( xmlTextReaderPtr& inputReader );
_Node _ReadXMLNode( xmlTextReaderPtr& inputReader );
double ApproximateDistance( const  int lat1, const  int lon1, const  int lat2, const  int lon2 );

_Stats stats;
Settings settings;
NodeMap AllNodes;

vector<NodeID> SignalNodes;
vector<NodeID> UsedNodes;
vector<_Way> UsedWays;

int main (int argc, char *argv[])
{
    if(argc <= 1)
    {
        cerr << "usage: " << endl << argv[0] << " <file.osm>" << endl;
        exit(-1);
    }
    cout << "reading input file. This may take some time ..." << flush;
    /*
     Default Speed Profile:
        motorway        120
        motorway_link   80
        trunk           100
        trunk_link      80
        secondary       100
        secondary_link  50
        primary         100
        primary_link    50
        tertiary        100
        unclassified    50
        residential     50
        living_street   30
        service         20
     */

    string names[13] = { "motorway", "motorway_link", "trunk", "trunk_link", "secondary", "secondary_link", "primary", "primary_link", "tertiary", "unclassified", "residential", "living_street", "service" };
    double speeds[13] = { 120, 80, 100, 80, 100, 50, 100, 50, 100, 50, 50 , 30, 20};
    settings.speedProfile.names.insert(settings.speedProfile.names.begin(), names, names+13);
    settings.speedProfile.speed.insert(settings.speedProfile.speed.begin(), speeds, speeds+13);

    AllNodes.set_empty_key(UINT_MAX);
    xmlTextReaderPtr inputReader = xmlNewTextReaderFilename( argv[1] );
    ofstream nodeFile("_nodes", ios::binary);
    ofstream wayFile("_ways", ios::binary);
    try {
        while ( xmlTextReaderRead( inputReader ) == 1 ) {
            const int type = xmlTextReaderNodeType( inputReader );

            //1 is Element
            if ( type != 1 )
                continue;

            xmlChar* currentName = xmlTextReaderName( inputReader );
            if ( currentName == NULL )
                continue;

            if ( xmlStrEqual( currentName, ( const xmlChar* ) "node" ) == 1 ) {
                stats.numberOfNodes++;
                _Node node = _ReadXMLNode( inputReader );
                AllNodes.insert(make_pair(node.id, node) );

                if ( node.trafficSignal )
                    SignalNodes.push_back( node.id );

            }
            else if ( xmlStrEqual( currentName, ( const xmlChar* ) "way" ) == 1 ) {
                stats.numberOfWays++;
                _Way way = _ReadXMLWay( inputReader );

                if ( way.usefull && way.access && way.path.size() ) {
                    for ( unsigned i = 0; i < way.path.size(); ++i ) {
                        UsedNodes.push_back( way.path[i] );
                    }

                    if ( way.direction == _Way::opposite )
                        std::reverse( way.path.begin(), way.path.end() );

                    stats.numberOfEdges += ( int ) way.path.size() - 1;

                    UsedWays.push_back(way);
                }
            }
            xmlFree( currentName );
        }
        sort(UsedNodes.begin(), UsedNodes.end());
        UsedNodes.erase(unique(UsedNodes.begin(), UsedNodes.end()), UsedNodes.end() );
        sort(SignalNodes.begin(), SignalNodes.end());
        SignalNodes.erase(unique(SignalNodes.begin(), SignalNodes.end()), SignalNodes.end() );
        cout << "ok" << endl;
        cout << endl << "Statistics: " << endl;
        cout << "All Nodes: " << stats.numberOfNodes << endl;
        cout << "Used Nodes: " << UsedNodes.size() << endl;
        cout << "Number of Ways: " << stats.numberOfWays << endl;
        cout << "Edges in graph: " << stats.numberOfEdges << endl;
        cout << "Number of ways with maxspeed information: " << stats.numberOfMaxspeed << endl;
        cout << "Number of nodes with traffic lights: " << SignalNodes.size() << endl;
        cout << "finished loading data" << endl;
        cout << "calculated edge weights and writing to disk ..." << flush;
        string name(argv[1]);
        int pos=name.find(".osm"); // pos=9
        if(pos!=string::npos)
        {
            //replace
            name.replace(pos, 5, ".osrm");
        } else {
            name.append(".osrm");
        }


        ofstream fout;
        fout.open(name.c_str());

        fout << UsedNodes.size() << endl;
        for(vector<NodeID>::size_type i = 0; i < UsedNodes.size(); i++)
        {
            NodeMap::iterator it = AllNodes.find(UsedNodes[i]);
            assert(it!=AllNodes.end());
            fout << UsedNodes[i] << " " << it->second.lon << " " << it->second.lat << "\n" << flush;
        }
        UsedNodes.clear();
        fout << stats.numberOfEdges << endl;
        for(vector<_Way>::size_type i = 0; i < UsedWays.size(); i++)
        {
            vector< NodeID > & path = UsedWays[i].path;
            double speed = UsedWays[i].maximumSpeed;
            assert(UsedWays[i].type > -1 || UsedWays[i].maximumSpeed != -1);
            assert(path.size()>0);

            for(vector< NodeID >::size_type n = 0; n < path.size()-1; n++)
            {
                //insert path[n], path[n+1]
                NodeMap::iterator startit = AllNodes.find(path[n]);
                if(startit == AllNodes.end())
                {
                    cerr << "Node " << path[n] << " missing albeit referenced in way. Edge skipped" << endl;
                    continue;<commit_msg>Incorrect usage of flushes fixed. Thanks to <NAME>.<commit_after>/*
    open source routing machine
    Copyright (C) <NAME>, others 2010

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU AFFERO General Public License as published by
the Free Software Foundation; either version 3 of the License, or
any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
or see http://www.gnu.org/licenses/agpl.txt.
 */

#include <cassert>
#include <climits>
#include <cstdlib>
#include <iostream>
#include <string>
#include <sstream>
#include <vector>
#include <libxml/xmlreader.h>

#include "typedefs.h"

using namespace std;

struct _Node : NodeInfo{
    bool trafficSignal:1;
};

struct _Way {
    std::vector< NodeID > path;
    enum {
        notSure = 0, oneway, bidirectional, opposite
    } direction;
    double maximumSpeed;
    bool usefull:1;
    bool access:1;
    short type;
};

typedef google::dense_hash_map<NodeID, _Node> NodeMap;

struct _Stats {
    NodeID numberOfNodes;
    NodeID numberOfEdges;
    NodeID numberOfWays;
    //    NodeID numberOfPlaces;
    //    NodeID numberOfOutlines;
    NodeID numberOfMaxspeed;
    //    NodeID numberOfZeroSpeed;
    //    NodeID numberOfDefaultCitySpeed;
    //    NodeID numberOfCityEdges;
};

struct Settings {
    struct SpeedProfile {
        vector< double > speed;
        vector< string > names;
    } speedProfile;
    vector<string> accessList;
    int trafficLightPenalty;
    int indexInAccessListOf( const string & key)
    {
        for(int i = 0; i< accessList.size(); i++)
        {
            if(accessList[i] == key)
                return i;
        }
        return -1;
    }
};

_Way _ReadXMLWay( xmlTextReaderPtr& inputReader );
_Node _ReadXMLNode( xmlTextReaderPtr& inputReader );
double ApproximateDistance( const  int lat1, const  int lon1, const  int lat2, const  int lon2 );

_Stats stats;
Settings settings;
NodeMap AllNodes;

vector<NodeID> SignalNodes;
vector<NodeID> UsedNodes;
vector<_Way> UsedWays;

int main (int argc, char *argv[])
{
    if(argc <= 1)
    {
        cerr << "usage: " << endl << argv[0] << " <file.osm>" << endl;
        exit(-1);
    }
    cout << "reading input file. This may take some time ..." << flush;
    /*
     Default Speed Profile:
        motorway        120
        motorway_link   80
        trunk           100
        trunk_link      80
        secondary       100
        secondary_link  50
        primary         100
        primary_link    50
        tertiary        100
        unclassified    50
        residential     50
        living_street   30
        service         20
     */

    string names[13] = { "motorway", "motorway_link", "trunk", "trunk_link", "secondary", "secondary_link", "primary", "primary_link", "tertiary", "unclassified", "residential", "living_street", "service" };
    double speeds[13] = { 120, 80, 100, 80, 100, 50, 100, 50, 100, 50, 50 , 30, 20};
    settings.speedProfile.names.insert(settings.speedProfile.names.begin(), names, names+13);
    settings.speedProfile.speed.insert(settings.speedProfile.speed.begin(), speeds, speeds+13);

    AllNodes.set_empty_key(UINT_MAX);
    xmlTextReaderPtr inputReader = xmlNewTextReaderFilename( argv[1] );
    ofstream nodeFile("_nodes", ios::binary);
    ofstream wayFile("_ways", ios::binary);
    try {
        while ( xmlTextReaderRead( inputReader ) == 1 ) {
            const int type = xmlTextReaderNodeType( inputReader );

            //1 is Element
            if ( type != 1 )
                continue;

            xmlChar* currentName = xmlTextReaderName( inputReader );
            if ( currentName == NULL )
                continue;

            if ( xmlStrEqual( currentName, ( const xmlChar* ) "node" ) == 1 ) {
                stats.numberOfNodes++;
                _Node node = _ReadXMLNode( inputReader );
                AllNodes.insert(make_pair(node.id, node) );

                if ( node.trafficSignal )
                    SignalNodes.push_back( node.id );

            }
            else if ( xmlStrEqual( currentName, ( const xmlChar* ) "way" ) == 1 ) {
                stats.numberOfWays++;
                _Way way = _ReadXMLWay( inputReader );

                if ( way.usefull && way.access && way.path.size() ) {
                    for ( unsigned i = 0; i < way.path.size(); ++i ) {
                        UsedNodes.push_back( way.path[i] );
                    }

                    if ( way.direction == _Way::opposite )
                        std::reverse( way.path.begin(), way.path.end() );

                    stats.numberOfEdges += ( int ) way.path.size() - 1;

                    UsedWays.push_back(way);
                }
            }
            xmlFree( currentName );
        }
        sort(UsedNodes.begin(), UsedNodes.end());
        UsedNodes.erase(unique(UsedNodes.begin(), UsedNodes.end()), UsedNodes.end() );
        sort(SignalNodes.begin(), SignalNodes.end());
        SignalNodes.erase(unique(SignalNodes.begin(), SignalNodes.end()), SignalNodes.end() );
        cout << "ok" << endl;
        cout << endl << "Statistics: " << endl;
        cout << "All Nodes: " << stats.numberOfNodes << endl;
        cout << "Used Nodes: " << UsedNodes.size() << endl;
        cout << "Number of Ways: " << stats.numberOfWays << endl;
        cout << "Edges in graph: " << stats.numberOfEdges << endl;
        cout << "Number of ways with maxspeed information: " << stats.numberOfMaxspeed << endl;
        cout << "Number of nodes with traffic lights: " << SignalNodes.size() << endl;
        cout << "finished loading data" << endl;
        cout << "calculated edge weights and writing to disk ..." << flush;
        string name(argv[1]);
        int pos=name.find(".osm"); // pos=9
        if(pos!=string::npos)
        {
            //replace
            name.replace(pos, 5, ".osrm");
        } else {
            name.append(".osrm");
        }


        ofstream fout;
        fout.open(name.c_str());

        fout << UsedNodes.size() << endl;
        for(vector<NodeID>::size_type i = 0; i < UsedNodes.size(); i++)
        {
            NodeMap::iterator it = AllNodes.find(UsedNodes[i]);
            assert(it!=AllNodes.end());
            fout << UsedNodes[i] << " " << it->second.lon << " " << it->second.lat << "\n";
        }
        fout << flush;
        UsedNodes.clear();
        fout << stats.numberOfEdges << endl;
        for(vector<_Way>::size_type i = 0; i < UsedWays.size(); i++)
        {
            vector< NodeID > & path = UsedWays[i].path;
            double speed = UsedWays[i].maximumSpeed;
            assert(UsedWays[i].type > -1 || UsedWays[i].maximumSpeed != -1);
            assert(path.size()>0);

            for(vector< NodeID >::size_type n = 0; n < path.size()-1; n++)
            {
                //insert path[n], path[n+1]
                NodeMap::iterator startit = AllNodes.find(path[n]);
                if(startit == AllNodes.end())
                {
                    cerr << "Node " << path[n] << " missing albeit referenced in way. Edge skipped" << endl;
                    continue;
{'fraction_non_alphanumeric': 0.12423476177801437, 'fraction_numerical': 0.016103273888741016, 'mean_word_length': 2.1395445999582203, 'pattern_counts': {'":': 0, '<': 256, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 30}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4467655', 'n_tokens_mistral': 4655, 'n_tokens_neox': 4329, 'n_words': 1205}