<commit_before>/***************************************************************************
**
** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).
** All rights reserved.
** Contact: Nokia Corporation (<EMAIL>)
**
** This file is part of libmeegotouch.
**
** If you have questions regarding the use of this file, please contact
** Nokia at <EMAIL>.
**
** This library is free software; you can redistribute it and/or
** modify it under the terms of the GNU Lesser General Public
** License version 2.1 as published by the Free Software Foundation
** and appearing in the file LICENSE.LGPL included in the packaging
** of this file.
**
****************************************************************************/

#include "mlocationdatabase.h"

#include <QFile>
#include <QTextStream>
#include <QDomDocument>
#include <QStringMatcher>
#include <QStringList>
#include <QDebug>

#ifdef HAVE_ICU
#include <unicode/timezone.h>
#endif

static const QString path = "/usr/share/meegotouch/locationdatabase/";
static const QString zoneAliasFile = "/usr/share/tzdata-timed/zone.alias";
static const QString zoneAliasFileFallback = ":/zone.alias.fallback";

class MLocationDatabasePrivate
{
public:
    MLocationDatabasePrivate();
    bool loadCountries();
    bool loadCities();
    bool loadTimeZoneData();
    QString canonicalizeTimeZoneId(QString timeZoneId);

    QHash<QString, MCity> cities;
    QHash<QString, MCountry> countries;
    QHash<QString, QString> canonicalTimeZoneIds;
};

MLocationDatabasePrivate::MLocationDatabasePrivate()
{
}

bool MLocationDatabasePrivate::loadCountries()
{
    QFile file( path + "countries.xml" );
    if (!file.open(QIODevice::ReadOnly))
    {
        qDebug( "loadCountries file open failed" );
        return false;
    }

    QDomDocument doc;
    if (!doc.setContent(&file))
    {
        qDebug( "loadCountries setContent failed" );
        file.close();
        return false;
    }
    
    file.close();

    QDomElement docElem = doc.documentElement();
    
    QDomNode n = docElem.firstChild();

    while(!n.isNull())
    {
        QDomElement e = n.toElement(); // try to convert the node to an element.
        if ( e.isNull() )
        {
            continue;
        }
        
        if ( e.tagName() != "country" )
        {
            continue;
        }

        // now we found a country key
        MCountry country;
        
        QDomElement tmpEl;

        tmpEl = e.elementsByTagName( "key" ).at( 0 ).toElement();
        country.setKey( tmpEl.text() );

        tmpEl = e.elementsByTagName( "englishname" ).at( 0 ).toElement();
        country.setEnglishName( tmpEl.text() );

        tmpEl = e.elementsByTagName( "localname" ).at( 0 ).toElement();
        country.setLocalName( tmpEl.text() );

        countries[ country.key() ] = country;

        // QTextStream s( stdout );
        // s.setCodec( "UTF-8" );

        // s        << "country: key: " << country.key()
        //          << "\nen:  " << country.englishName()
        //          << "\nloc: " << country.localName() << "\n";

        n = n.nextSibling();
    }

    return true;
}


bool MLocationDatabasePrivate::loadCities()
{
    QFile file( path + "cities.xml" );
    if (!file.open(QIODevice::ReadOnly))
    {
        qDebug( "loadCitiess file open failed" );
        return false;
    }

    QDomDocument doc;
    if (!doc.setContent(&file))
    {
        qDebug( "loadCities setContent failed" );
        file.close();
        return false;
    }
    
    file.close();

    QDomElement docElem = doc.documentElement();
    
    QDomNode n = docElem.firstChild();

    while(!n.isNull())
    {
        QDomElement e = n.toElement(); // try to convert the node to an element.
        if ( e.isNull() )
        {
            continue;
        }
        
        if ( e.tagName() != "city" )
        {<commit_msg>Changes: fix typo in debug message<commit_after>/***************************************************************************
**
** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).
** All rights reserved.
** Contact: Nokia Corporation (<EMAIL>)
**
** This file is part of libmeegotouch.
**
** If you have questions regarding the use of this file, please contact
** Nokia at <EMAIL>.
**
** This library is free software; you can redistribute it and/or
** modify it under the terms of the GNU Lesser General Public
** License version 2.1 as published by the Free Software Foundation
** and appearing in the file LICENSE.LGPL included in the packaging
** of this file.
**
****************************************************************************/

#include "mlocationdatabase.h"

#include <QFile>
#include <QTextStream>
#include <QDomDocument>
#include <QStringMatcher>
#include <QStringList>
#include <QDebug>

#ifdef HAVE_ICU
#include <unicode/timezone.h>
#endif

static const QString path = "/usr/share/meegotouch/locationdatabase/";
static const QString zoneAliasFile = "/usr/share/tzdata-timed/zone.alias";
static const QString zoneAliasFileFallback = ":/zone.alias.fallback";

class MLocationDatabasePrivate
{
public:
    MLocationDatabasePrivate();
    bool loadCountries();
    bool loadCities();
    bool loadTimeZoneData();
    QString canonicalizeTimeZoneId(QString timeZoneId);

    QHash<QString, MCity> cities;
    QHash<QString, MCountry> countries;
    QHash<QString, QString> canonicalTimeZoneIds;
};

MLocationDatabasePrivate::MLocationDatabasePrivate()
{
}

bool MLocationDatabasePrivate::loadCountries()
{
    QFile file( path + "countries.xml" );
    if (!file.open(QIODevice::ReadOnly))
    {
        qDebug( "loadCountries file open failed" );
        return false;
    }

    QDomDocument doc;
    if (!doc.setContent(&file))
    {
        qDebug( "loadCountries setContent failed" );
        file.close();
        return false;
    }
    
    file.close();

    QDomElement docElem = doc.documentElement();
    
    QDomNode n = docElem.firstChild();

    while(!n.isNull())
    {
        QDomElement e = n.toElement(); // try to convert the node to an element.
        if ( e.isNull() )
        {
            continue;
        }
        
        if ( e.tagName() != "country" )
        {
            continue;
        }

        // now we found a country key
        MCountry country;
        
        QDomElement tmpEl;

        tmpEl = e.elementsByTagName( "key" ).at( 0 ).toElement();
        country.setKey( tmpEl.text() );

        tmpEl = e.elementsByTagName( "englishname" ).at( 0 ).toElement();
        country.setEnglishName( tmpEl.text() );

        tmpEl = e.elementsByTagName( "localname" ).at( 0 ).toElement();
        country.setLocalName( tmpEl.text() );

        countries[ country.key() ] = country;

        // QTextStream s( stdout );
        // s.setCodec( "UTF-8" );

        // s        << "country: key: " << country.key()
        //          << "\nen:  " << country.englishName()
        //          << "\nloc: " << country.localName() << "\n";

        n = n.nextSibling();
    }

    return true;
}


bool MLocationDatabasePrivate::loadCities()
{
    QFile file( path + "cities.xml" );
    if (!file.open(QIODevice::ReadOnly))
    {
        qDebug( "loadCities file open failed" );
        return false;
    }

    QDomDocument doc;
    if (!doc.setContent(&file))
    {
        qDebug( "loadCities setContent failed" );
        file.close();
        return false;
    }
    
    file.close();

    QDomElement docElem = doc.documentElement();
    
    QDomNode n = docElem.firstChild();

    while(!n.isNull())
    {
        QDomElement e = n.toElement(); // try to convert the node to an element.
        if ( e.isNull() )
        {
            continue;
        }
        
        if ( e.tagName() != "city" )
        {
{'fraction_non_alphanumeric': 0.16353092783505155, 'fraction_numerical': 0.002577319587628866, 'mean_word_length': 2.5930555555555554, 'pattern_counts': {'":': 2, '<': 55, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '511034', 'n_tokens_mistral': 2345, 'n_tokens_neox': 2264, 'n_words': 597}