<commit_before>/* @(#)dtkDistributedPolicy.cpp ---
 *
 * Author: <NAME>
 * Copyright (C) 2013 - <NAME>, Inria.
 * Created: 2013/02/18 15:18:41
 */

/* Commentary:
 *
 */

/* Change log:
 *
 */

#include <dtkCore/dtkCore.h>

#include "dtkDistributedCoreApplication.h"
#include "dtkDistributedPolicy.h"

#include <dtkDistributed>
#include <dtkLog>

#include <QCommandLineParser>

class dtkDistributedPolicyPrivate
{
public:
    QStringList hosts;

public:
    qlonglong nthreads;
    QString type;

public:
    dtkDistributedCommunicator *comm;
};

dtkDistributedPolicy::dtkDistributedPolicy(void) : QObject(), d(new dtkDistributedPolicyPrivate)
{
    d->comm = NULL;
    d->nthreads = 1;
    d->type = "qthread";
}

dtkDistributedPolicy::~dtkDistributedPolicy(void)
{
    delete d;

    d = NULL;
}

dtkDistributedPolicy::dtkDistributedPolicy(const dtkDistributedPolicy& other)
{
}

dtkDistributedPolicy& dtkDistributedPolicy::operator = (const dtkDistributedPolicy& other)
{
    return (*this);
}

void dtkDistributedPolicy::addHost(QString host)
{
    dtkTrace() << "add host " << host;

    d->hosts.append(host);
}

dtkDistributedCommunicator *dtkDistributedPolicy::communicator(void)
{
    return d->comm;
}

void dtkDistributedPolicy::setType(const QString& type)
{
    qDebug() << "create" << type << "communicator";
    d->type = type;
    d->comm = dtkDistributed::communicator::pluginFactory().create(type);
}

QStringList dtkDistributedPolicy::types(void)
{
    return dtkDistributed::communicator::pluginFactory().keys();
}

QStringList dtkDistributedPolicy::hosts(void)
{
    if (d->hosts.count() == 0) {
        //Try to get hostsfile from env
        QStringList schedulers;
        schedulers << "PBS_NODEFILE";
        schedulers << "OAR_NODEFILE";
        foreach (QString envname, schedulers) {
            QString nodefile =  QString::fromUtf8(qgetenv(qPrintable(envname)));
            if (!nodefile.isEmpty()) {
               dtkDebug() << "Extracting hosts from file" << nodefile;
                QFile file(nodefile);
                if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
                    dtkWarn() << "Error while opening"<< nodefile;
                    return d->hosts;
                }
                QTextStream in(&file);
                while (!in.atEnd()) {
                    d->hosts <<  in.readLine();
                }
                if (d->type == "hybrid") {
                    d->nthreads = d->hosts.count();
                    d->hosts.removeDuplicates();
                    d->nthreads /= d->hosts.count();
                }
                return d->hosts;
            }
        }
        dtkDebug() << "No hostfile found, try qapp args";
        d->nthreads = 1;
// skip this part on windows with qt < 5.5 because of QTBUG-30330
#if QT_VERSION < 0x050500 && defined Q_OS_WIN32
        dtkWarn() << "skip running parser on windows";
#else
        QCommandLineParser *parser = dtkDistributed::app()->parser();
        QCommandLineOption npOption("np","number of processes","int");
        QCommandLineOption ntOption("nt","number of threads","int");
        parser->process(qApp->arguments());
        int np = 1;
        if (parser->isSet(npOption)) {
            dtkTrace() << "got np value from command line" ;
            np = parser->value(npOption).toInt();
        } else {
            QByteArray numprocs = qgetenv("DTK_NUM_THREADS");
            if (!numprocs.isEmpty()) {
                np = numprocs.toInt();
                dtkDebug() << "got num procs from env" << np;
            }
        }
        for (int i = 0; i <  np; i++) {
            d->hosts <<  "localhost";
        }
        //FIXME: rework hybrid case
        if (parser->isSet(ntOption)) {
                if (d->type == "mpi") {
                    int nt = parser->value(ntOption).toInt();
                    int total = d->hosts.count() * nt;
                    for (int i = d->hosts.count(); i <  total; i++) {
                        d->hosts <<  "localhost";
                    }<commit_msg>we should not return an empty host list<commit_after>/* @(#)dtkDistributedPolicy.cpp ---
 *
 * Author: <NAME>
 * Copyright (C) 2013 - <NAME>, Inria.
 * Created: 2013/02/18 15:18:41
 */

/* Commentary:
 *
 */

/* Change log:
 *
 */

#include <dtkCore/dtkCore.h>

#include "dtkDistributedCoreApplication.h"
#include "dtkDistributedPolicy.h"

#include <dtkDistributed>
#include <dtkLog>

#include <QCommandLineParser>

class dtkDistributedPolicyPrivate
{
public:
    QStringList hosts;

public:
    qlonglong nthreads;
    QString type;

public:
    dtkDistributedCommunicator *comm;
};

dtkDistributedPolicy::dtkDistributedPolicy(void) : QObject(), d(new dtkDistributedPolicyPrivate)
{
    d->comm = NULL;
    d->nthreads = 1;
    d->type = "qthread";
}

dtkDistributedPolicy::~dtkDistributedPolicy(void)
{
    delete d;

    d = NULL;
}

dtkDistributedPolicy::dtkDistributedPolicy(const dtkDistributedPolicy& other)
{
}

dtkDistributedPolicy& dtkDistributedPolicy::operator = (const dtkDistributedPolicy& other)
{
    return (*this);
}

void dtkDistributedPolicy::addHost(QString host)
{
    dtkTrace() << "add host " << host;

    d->hosts.append(host);
}

dtkDistributedCommunicator *dtkDistributedPolicy::communicator(void)
{
    return d->comm;
}

void dtkDistributedPolicy::setType(const QString& type)
{
    qDebug() << "create" << type << "communicator";
    d->type = type;
    d->comm = dtkDistributed::communicator::pluginFactory().create(type);
}

QStringList dtkDistributedPolicy::types(void)
{
    return dtkDistributed::communicator::pluginFactory().keys();
}

QStringList dtkDistributedPolicy::hosts(void)
{
    if (d->hosts.count() == 0) {
        //Try to get hostsfile from env
        QStringList schedulers;
        schedulers << "PBS_NODEFILE";
        schedulers << "OAR_NODEFILE";
        foreach (QString envname, schedulers) {
            QString nodefile =  QString::fromUtf8(qgetenv(qPrintable(envname)));
            if (!nodefile.isEmpty()) {
               dtkDebug() << "Extracting hosts from file" << nodefile;
                QFile file(nodefile);
                if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
                    dtkWarn() << "Error while opening"<< nodefile;
                    return d->hosts;
                }
                QTextStream in(&file);
                while (!in.atEnd()) {
                    d->hosts <<  in.readLine();
                }
                if (d->type == "hybrid") {
                    d->nthreads = d->hosts.count();
                    d->hosts.removeDuplicates();
                    d->nthreads /= d->hosts.count();
                }
                return d->hosts;
            }
        }
        dtkDebug() << "No hostfile found, try qapp args";
        d->nthreads = 1;
// skip this part on windows with qt < 5.5 because of QTBUG-30330
#if QT_VERSION < 0x050500 && defined Q_OS_WIN32
        dtkWarn() << "skip running parser on windows";
        d->hosts <<  "localhost";
#else
        QCommandLineParser *parser = dtkDistributed::app()->parser();
        QCommandLineOption npOption("np","number of processes","int");
        QCommandLineOption ntOption("nt","number of threads","int");
        parser->process(qApp->arguments());
        int np = 1;
        if (parser->isSet(npOption)) {
            dtkTrace() << "got np value from command line" ;
            np = parser->value(npOption).toInt();
        } else {
            QByteArray numprocs = qgetenv("DTK_NUM_THREADS");
            if (!numprocs.isEmpty()) {
                np = numprocs.toInt();
                dtkDebug() << "got num procs from env" << np;
            }
        }
        for (int i = 0; i <  np; i++) {
            d->hosts <<  "localhost";
        }
        //FIXME: rework hybrid case
        if (parser->isSet(ntOption)) {
                if (d->type == "mpi") {
                    int nt = parser->value(ntOption).toInt();
                    int total = d->hosts.count() * nt;
                    for (int i = d->hosts.count(); i <  total; i++) {
                        d->hosts <<  "localhost";
                    }
{'fraction_non_alphanumeric': 0.13982843137254902, 'fraction_numerical': 0.00980392156862745, 'mean_word_length': 2.2269671807038356, 'pattern_counts': {'":': 0, '<': 101, '<?xml version=': 0, '>': 76, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2756151', 'n_tokens_mistral': 2637, 'n_tokens_neox': 2504, 'n_words': 529}