<commit_before>/*
    open source routing machine
    Copyright (C) <NAME>, others 2010

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU AFFERO General Public License as published by
the Free Software Foundation; either version 3 of the License, or
any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
or see http://www.gnu.org/licenses/agpl.txt.
 */

#ifndef XMLPARSER_H_
#define XMLPARSER_H_

#include <libxml/xmlreader.h>

#include "../typedefs.h"
#include "BaseParser.h"
#include "HashTable.h"
#include "ExtractorStructs.h"
#include "InputReaderFactory.h"

class XMLParser : public BaseParser<_Node, _RawRestrictionContainer, _Way> {
public:
    XMLParser(const char * filename) {
        WARN("Parsing plain .osm/.osm.bz2 is deprecated. Switch to .pbf");
        inputReader = inputReaderFactory(filename);
    }
    virtual ~XMLParser() {}

    bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*restrictionCallbackPointer)(_RawRestrictionContainer), bool (*wayCallbackPointer)(_Way), bool (*addressCallbackPointer)(_Node, HashTable<std::string, std::string>) ) {
        nodeCallback = *nodeCallbackPointer;
        wayCallback = *wayCallbackPointer;
        restrictionCallback = *restrictionCallbackPointer;
        return true;
    }
    bool Init() {
        return (xmlTextReaderRead( inputReader ) == 1);
    }
    bool Parse() {
        while ( xmlTextReaderRead( inputReader ) == 1 ) {
            const int type = xmlTextReaderNodeType( inputReader );

            //1 is Element
            if ( type != 1 )
                continue;

            xmlChar* currentName = xmlTextReaderName( inputReader );
            if ( currentName == NULL )
                continue;

            if ( xmlStrEqual( currentName, ( const xmlChar* ) "node" ) == 1 ) {
                _Node n = _ReadXMLNode(  );
                if(!(*nodeCallback)(n))
                    std::cerr << "[XMLParser] node not parsed" << std::endl;
            }

            if ( xmlStrEqual( currentName, ( const xmlChar* ) "way" ) == 1 ) {
                string name;
                _Way way = _ReadXMLWay( );
                if(!(*wayCallback)(way)) {
                    std::cerr << "[XMLParser] way not parsed" << std::endl;
                }
            }
            if ( xmlStrEqual( currentName, ( const xmlChar* ) "relation" ) == 1 ) {
                _Relation r;
                r.type = _Relation::unknown;
                //todo: parse relation
            }
            xmlFree( currentName );
        }
        return true;
    }

private:
    _Relation _ReadXMLRelation ( ) {
        _Relation relation;
        relation.type = _Relation::unknown;

        return relation;
    }

    _Way _ReadXMLWay( ) {
        _Way way;
        way.direction = _Way::notSure;
        way.speed = -1;
        way.type = -1;
        way.useful = false;
        way.access = true;
        //    cout << "new way" << endl;
        if ( xmlTextReaderIsEmptyElement( inputReader ) != 1 ) {
            const int depth = xmlTextReaderDepth( inputReader );
            while ( xmlTextReaderRead( inputReader ) == 1 ) {
                const int childType = xmlTextReaderNodeType( inputReader );
                if ( childType != 1 && childType != 15 )
                    continue;
                const int childDepth = xmlTextReaderDepth( inputReader );
                xmlChar* childName = xmlTextReaderName( inputReader );
                if ( childName == NULL )
                    continue;

                if ( depth == childDepth && childType == 15 && xmlStrEqual( childName, ( const xmlChar* ) "way" ) == 1 ) {
                    xmlFree( childName );
                    break;
                }
                if ( childType != 1 ) {
                    xmlFree( childName );
                    continue;
                }

                if ( xmlStrEqual( childName, ( const xmlChar* ) "tag" ) == 1 ) {
                    xmlChar* k = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "k" );
                    xmlChar* value = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "v" );
                    //                cout << "->k=" << k << ", v=" << value << endl;
                    if ( k != NULL && value != NULL ) {

                        way.keyVals.Add(std::string( (char *) k ), std::string( (char *) value));
                    }
                    if ( k != NULL )
                        xmlFree( k );
                    if ( value != NULL )
                        xmlFree( value );
                } else if ( xmlStrEqual( childName, ( const xmlChar* ) "nd" ) == 1 ) {
                    xmlChar* ref = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "ref" );
                    if ( ref != NULL ) {
                        way.path.push_back( atoi(( const char* ) ref ) );
                        xmlFree( ref );
                    }
                }
                xmlFree( childName );
            }
        }
        return way;
    }


    _Node _ReadXMLNode( ) {
        _Node node;

        xmlChar* attribute = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "lat" );
        if ( attribute != NULL ) {
            node.lat =  static_cast<NodeID>(100000.*atof(( const char* ) attribute ) );
            xmlFree( attribute );
        }
        attribute = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "lon" );
        if ( attribute != NULL ) {
            node.lon =  static_cast<NodeID>(100000.*atof(( const char* ) attribute ));
            xmlFree( attribute );
        }
        attribute = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "id" );
        if ( attribute != NULL ) {
            node.id =  atoi(( const char* ) attribute );
            xmlFree( attribute );
        }

        if ( xmlTextReaderIsEmptyElement( inputReader ) != 1 ) {
            const int depth = xmlTextReaderDepth( inputReader );
            while ( xmlTextReaderRead( inputReader ) == 1 ) {
                const int childType = xmlTextReaderNodeType( inputReader );
                // 1 = Element, 15 = EndElement
                if ( childType != 1 && childType != 15 )
                    continue;
                const int childDepth = xmlTextReaderDepth( inputReader );
                xmlChar* childName = xmlTextReaderName( inputReader );
                if ( childName == NULL )
                    continue;

                if ( depth == childDepth && childType == 15 && xmlStrEqual( childName, ( const xmlChar* ) "node" ) == 1 ) {
                    xmlFree( childName );
                    break;
                }
                if ( childType != 1 ) {
                    xmlFree( childName );
                    continue;
                }

                if ( xmlStrEqual( childName, ( const xmlChar* ) "tag" ) == 1 ) {
                    xmlChar* k = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "k" );
                    xmlChar* value = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "v" );
                    if ( k != NULL && value != NULL ) {
                        if ( xmlStrEqual( k, ( const xmlChar* ) "highway" ) == 1 ) {
                            if ( xmlStrEqual( value, ( const xmlChar* ) "traffic_signals" ) == 1 ){
                                //node.trafficSignal = true;
                            }
                        }
                    }
                    if ( k != NULL )
                        xmlFree( k );
                    if ( value != NULL )
                        xmlFree( value );
                }

                xmlFree( childName );
            }
        }
        return node;
    }
    /* Input Reader */
    xmlTextReaderPtr inputReader;

    /* Function pointer for nodes */
    bool (*nodeCallback)(_Node);
    bool (*wayCallback)(_Way);
    bool (*restrictionCallback)(_RawRestrictionContainer);

};

#endif /* XMLPARSER_H_ */
<commit_msg>Bringing XML parser up to speed and adding support for turn restriction parsing. Note that it will be removed after the release of 0.3<commit_after>/*
    open source routing machine
    Copyright (C) <NAME>, others 2010

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU AFFERO General Public License as published by
the Free Software Foundation; either version 3 of the License, or
any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
or see http://www.gnu.org/licenses/agpl.txt.
 */

#ifndef XMLPARSER_H_
#define XMLPARSER_H_

#include <libxml/xmlreader.h>

#include "../typedefs.h"
#include "BaseParser.h"
#include "HashTable.h"
#include "ExtractorStructs.h"
#include "InputReaderFactory.h"

class XMLParser : public BaseParser<_Node, _RawRestrictionContainer, _Way> {
public:
    XMLParser(const char * filename) : nodeCallback(NULL), wayCallback(NULL), restrictionCallback(NULL){
        WARN("Parsing plain .osm/.osm.bz2 is deprecated. Switch to .pbf");
        inputReader = inputReaderFactory(filename);
    }
    virtual ~XMLParser() {}

    bool RegisterCallbacks(bool (*nodeCallbackPointer)(_Node), bool (*restrictionCallbackPointer)(_RawRestrictionContainer), bool (*wayCallbackPointer)(_Way), bool (*addressCallbackPointer)(_Node, HashTable<std::string, std::string>) ) {
        nodeCallback = *nodeCallbackPointer;
        wayCallback = *wayCallbackPointer;
        restrictionCallback = *restrictionCallbackPointer;
        return true;
    }
    bool Init() {
        return (xmlTextReaderRead( inputReader ) == 1);
    }
    bool Parse() {
        while ( xmlTextReaderRead( inputReader ) == 1 ) {
            const int type = xmlTextReaderNodeType( inputReader );

            //1 is Element
            if ( type != 1 )
                continue;

            xmlChar* currentName = xmlTextReaderName( inputReader );
            if ( currentName == NULL )
                continue;

            if ( xmlStrEqual( currentName, ( const xmlChar* ) "node" ) == 1 ) {
                _Node n = _ReadXMLNode(  );
                if(!(*nodeCallback)(n))
                    std::cerr << "[XMLParser] node not parsed" << std::endl;
            }

            if ( xmlStrEqual( currentName, ( const xmlChar* ) "way" ) == 1 ) {
                string name;
                _Way way = _ReadXMLWay( );
                if(!(*wayCallback)(way)) {
                    std::cerr << "[XMLParser] way not parsed" << std::endl;
                }
            }
            if ( xmlStrEqual( currentName, ( const xmlChar* ) "relation" ) == 1 ) {
                _RawRestrictionContainer r = _ReadXMLRestriction();
                if(r.fromWay != UINT_MAX) {
                    if(!(*restrictionCallback)(r)) {
                        std::cerr << "[XMLParser] restriction not parsed" << std::endl;
                    }
                }
            }
            xmlFree( currentName );
        }
        return true;
    }

private:
    _RawRestrictionContainer _ReadXMLRestriction ( ) {
        _RawRestrictionContainer restriction;

        if ( xmlTextReaderIsEmptyElement( inputReader ) != 1 ) {
            const int depth = xmlTextReaderDepth( inputReader );while ( xmlTextReaderRead( inputReader ) == 1 ) {
                const int childType = xmlTextReaderNodeType( inputReader );
                if ( childType != 1 && childType != 15 )
                    continue;
                const int childDepth = xmlTextReaderDepth( inputReader );
                xmlChar* childName = xmlTextReaderName( inputReader );
                if ( childName == NULL )
                    continue;

                if ( depth == childDepth && childType == 15 && xmlStrEqual( childName, ( const xmlChar* ) "relation" ) == 1 ) {
                    xmlFree( childName );
                    break;
                }
                if ( childType != 1 ) {
                    xmlFree( childName );
                    continue;
                }

                if ( xmlStrEqual( childName, ( const xmlChar* ) "tag" ) == 1 ) {
                    xmlChar* k = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "k" );
                    xmlChar* value = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "v" );
                    if ( k != NULL && value != NULL ) {
                        if(xmlStrEqual(k, ( const xmlChar* ) "restriction" )){
                            if(0 == std::string((const char *) value).find("only_"))
                                restriction.restriction.flags.isOnly = true;
                        }

                    }
                    if ( k != NULL )
                        xmlFree( k );
                    if ( value != NULL )
                        xmlFree( value );
                } else if ( xmlStrEqual( childName, ( const xmlChar* ) "member" ) == 1 ) {
                    xmlChar* ref = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "ref" );
                    if ( ref != NULL ) {
                        xmlChar * role = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "role" );
                        xmlChar * type = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "type" );
                        if(xmlStrEqual(role, (const xmlChar *) "to") && xmlStrEqual(type, (const xmlChar *) "way")) {
                            restriction.toWay = atoi((const char*) ref);
                        }
                        if(xmlStrEqual(role, (const xmlChar *) "from") && xmlStrEqual(type, (const xmlChar *) "way")) {
                            restriction.fromWay = atoi((const char*) ref);
                        }
                        if(xmlStrEqual(role, (const xmlChar *) "via") && xmlStrEqual(type, (const xmlChar *) "node")) {
                            restriction.restriction.viaNode = atoi((const char*) ref);
                        }

                        if(NULL != type)
                            xmlFree( type );
                        if(NULL != role)
                            xmlFree( role );
                        if(NULL != ref)
                            xmlFree( ref );
                    }
                }
                xmlFree( childName );
            }
        }
        return restriction;
    }

    _Way _ReadXMLWay( ) {
        _Way way;
        way.direction = _Way::notSure;
        way.speed = -1;
        way.type = -1;
        way.useful = false;
        way.access = true;
        //    cout << "new way" << endl;
        if ( xmlTextReaderIsEmptyElement( inputReader ) != 1 ) {
            const int depth = xmlTextReaderDepth( inputReader );
            while ( xmlTextReaderRead( inputReader ) == 1 ) {
                const int childType = xmlTextReaderNodeType( inputReader );
                if ( childType != 1 && childType != 15 )
                    continue;
                const int childDepth = xmlTextReaderDepth( inputReader );
                xmlChar* childName = xmlTextReaderName( inputReader );
                if ( childName == NULL )
                    continue;

                if ( depth == childDepth && childType == 15 && xmlStrEqual( childName, ( const xmlChar* ) "way" ) == 1 ) {
                    xmlChar* id = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "id" );
                    way.id = atoi((char*)id);
                    xmlFree(id);
                    xmlFree( childName );
                    break;
                }
                if ( childType != 1 ) {
                    xmlFree( childName );
                    continue;
                }

                if ( xmlStrEqual( childName, ( const xmlChar* ) "tag" ) == 1 ) {
                    xmlChar* k = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "k" );
                    xmlChar* value = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "v" );
                    //                cout << "->k=" << k << ", v=" << value << endl;
                    if ( k != NULL && value != NULL ) {

                        way.keyVals.Add(std::string( (char *) k ), std::string( (char *) value));
                    }
                    if ( k != NULL )
                        xmlFree( k );
                    if ( value != NULL )
                        xmlFree( value );
                } else if ( xmlStrEqual( childName, ( const xmlChar* ) "nd" ) == 1 ) {
                    xmlChar* ref = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "ref" );
                    if ( ref != NULL ) {
                        way.path.push_back( atoi(( const char* ) ref ) );
                        xmlFree( ref );
                    }
                }
                xmlFree( childName );
            }
        }
        return way;
    }


    _Node _ReadXMLNode( ) {
        _Node node;

        xmlChar* attribute = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "lat" );
        if ( attribute != NULL ) {
            node.lat =  static_cast<NodeID>(100000.*atof(( const char* ) attribute ) );
            xmlFree( attribute );
        }
        attribute = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "lon" );
        if ( attribute != NULL ) {
            node.lon =  static_cast<NodeID>(100000.*atof(( const char* ) attribute ));
            xmlFree( attribute );
        }
        attribute = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "id" );
        if ( attribute != NULL ) {
            node.id =  atoi(( const char* ) attribute );
            xmlFree( attribute );
        }

        if ( xmlTextReaderIsEmptyElement( inputReader ) != 1 ) {
            const int depth = xmlTextReaderDepth( inputReader );
            while ( xmlTextReaderRead( inputReader ) == 1 ) {
                const int childType = xmlTextReaderNodeType( inputReader );
                // 1 = Element, 15 = EndElement
                if ( childType != 1 && childType != 15 )
                    continue;
                const int childDepth = xmlTextReaderDepth( inputReader );
                xmlChar* childName = xmlTextReaderName( inputReader );
                if ( childName == NULL )
                    continue;

                if ( depth == childDepth && childType == 15 && xmlStrEqual( childName, ( const xmlChar* ) "node" ) == 1 ) {
                    xmlFree( childName );
                    break;
                }
                if ( childType != 1 ) {
                    xmlFree( childName );
                    continue;
                }

                if ( xmlStrEqual( childName, ( const xmlChar* ) "tag" ) == 1 ) {
                    xmlChar* k = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "k" );
                    xmlChar* value = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "v" );
                    if ( k != NULL && value != NULL ) {
                        if ( xmlStrEqual( k, ( const xmlChar* ) "highway" ) == 1 ) {
                            if ( xmlStrEqual( value, ( const xmlChar* ) "traffic_signals" ) == 1 ){
                                //node.trafficSignal = true;
                            }
                        }
                    }
                    if ( k != NULL )
                        xmlFree( k );
                    if ( value != NULL )
                        xmlFree( value );
                }

                xmlFree( childName );
            }
        }
        return node;
    }
    /* Input Reader */
    xmlTextReaderPtr inputReader;

    /* Function pointer for nodes */
    bool (*nodeCallback)(_Node);
    bool (*wayCallback)(_Way);
    bool (*restrictionCallback)(_RawRestrictionContainer);

};

#endif /* XMLPARSER_H_ */

{'fraction_non_alphanumeric': 0.10631051428015374, 'fraction_numerical': 0.007200895246436043, 'mean_word_length': 1.5269240226210967, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 212}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6480303', 'n_tokens_mistral': 5534, 'n_tokens_neox': 5109, 'n_words': 1454}