<commit_before>/****************************************************************************
**
** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
** Contact: Qt Software Information (<EMAIL>)
**
** This file is part of the Qt Linguist of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** No Commercial Usage
** This file contains pre-release code and may not be distributed.
** You may use this file in accordance with the terms and conditions
** contained in the either Technology Preview License Agreement or the
** Beta Release License Agreement.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Nokia gives you certain
** additional rights. These rights are described in the Nokia Qt LGPL
** Exception version 1.0, included in the file LGPL_EXCEPTION.txt in this
** package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
** If you are unsure which license is appropriate for your use, please
** contact the sales department at <EMAIL>.
** $QT_END_LICENSE$
**
****************************************************************************/

#include "profileevaluator.h"
#include "proparserutils.h"
#include "proitems.h"

#include <QtCore/QByteArray>
#include <QtCore/QDateTime>
#include <QtCore/QDebug>
#include <QtCore/QDir>
#include <QtCore/QFile>
#include <QtCore/QFileInfo>
#include <QtCore/QList>
#include <QtCore/QRegExp>
#include <QtCore/QSet>
#include <QtCore/QStack>
#include <QtCore/QString>
#include <QtCore/QStringList>
#include <QtCore/QTextStream>

#ifdef Q_OS_UNIX
#include <unistd.h>
#include <sys/utsname.h>
#elif defined(Q_OS_WIN32)
#include <Windows.h>
#endif
#include <stdio.h>
#include <stdlib.h>

#ifdef Q_OS_WIN32
#define QT_POPEN _popen
#else
#define QT_POPEN popen
#endif

QT_BEGIN_NAMESPACE

///////////////////////////////////////////////////////////////////////
//
// Option
//
///////////////////////////////////////////////////////////////////////

QString
Option::fixString(QString string, uchar flags)
{
    // XXX Ripped out caching, so this will be slow. Should not matter for current uses.

    //fix the environment variables
    if (flags & Option::FixEnvVars) {
        int rep;
        QRegExp reg_variableName(QLatin1String("\\$\\(.*\\)"));
        reg_variableName.setMinimal(true);
        while ((rep = reg_variableName.indexIn(string)) != -1)
            string.replace(rep, reg_variableName.matchedLength(),
                           QString::fromLocal8Bit(qgetenv(string.mid(rep + 2, reg_variableName.matchedLength() - 3).toLatin1().constData()).constData()));
    }

    //canonicalize it (and treat as a path)
    if (flags & Option::FixPathCanonicalize) {
#if 0
        string = QFileInfo(string).canonicalFilePath();
#endif
        string = QDir::cleanPath(string);
    }

    if (string.length() > 2 && string[0].isLetter() && string[1] == QLatin1Char(':'))
        string[0] = string[0].toLower();

    //fix separators
    Q_ASSERT(!((flags & Option::FixPathToLocalSeparators) && (flags & Option::FixPathToTargetSeparators)));
    if (flags & Option::FixPathToLocalSeparators) {
#if defined(Q_OS_WIN32)
        string = string.replace(QLatin1Char('/'), QLatin1Char('\\'));
#else
        string = string.replace(QLatin1Char('\\'), QLatin1Char('/'));
#endif
    } else if (flags & Option::FixPathToTargetSeparators) {
        string = string.replace(QLatin1Char('/'), Option::dir_sep)
                       .replace(QLatin1Char('\\'), Option::dir_sep);
    }

    if ((string.startsWith(QLatin1Char('"')) && string.endsWith(QLatin1Char('"'))) ||
        (string.startsWith(QLatin1Char('\'')) && string.endsWith(QLatin1Char('\''))))
        string = string.mid(1, string.length() - 2);

    return string;
}

///////////////////////////////////////////////////////////////////////
//
// ProFileEvaluator::Private
//
///////////////////////////////////////////////////////////////////////

class ProFileEvaluator::Private : public AbstractProItemVisitor
{
public:
    Private(ProFileEvaluator *q_);

    ProFileEvaluator *q;
    int m_lineNo;                                   // Error reporting
    bool m_verbose;

    /////////////// Reading pro file

    bool read(ProFile *pro);

    ProBlock *currentBlock();
    void updateItem();
    bool parseLine(const QString &line);
    void insertVariable(const QString &line, int *i);
    void insertOperator(const char op);
    void insertComment(const QString &comment);
    void enterScope(bool multiLine);
    void leaveScope();
    void finalizeBlock();

    QStack<ProBlock *> m_blockstack;
    ProBlock *m_block;

    ProItem *m_commentItem;
    QString m_proitem;
    QString m_pendingComment;
    bool m_syntaxError;
    bool m_contNextLine;

    /////////////// Evaluating pro file contents

    // implementation of AbstractProItemVisitor
    bool visitBeginProBlock(ProBlock *block);
    bool visitEndProBlock(ProBlock *block);
    bool visitBeginProVariable(ProVariable *variable);
    bool visitEndProVariable(ProVariable *variable);
    bool visitBeginProFile(ProFile *value);
    bool visitEndProFile(ProFile *value);
    bool visitProValue(ProValue *value);
    bool visitProFunction(ProFunction *function);
    bool visitProOperator(ProOperator *oper);
    bool visitProCondition(ProCondition *condition);

    QStringList valuesDirect(const QString &variableName) const { return m_valuemap[variableName]; }
    QStringList values(const QString &variableName) const;
    QStringList values(const QString &variableName, const ProFile *pro) const;
    QStringList values(const QString &variableName, const QHash<QString, QStringList> &place,
                       const ProFile *pro) const;
    QString propertyValue(const QString &val) const;

    bool isActiveConfig(const QString &config, bool regex = false);
    QStringList expandPattern(const QString &pattern);
    void expandPatternHelper(const QString &relName, const QString &absName,
        QStringList &sources_out);
    QStringList expandVariableReferences(const QString &value);
    QStringList evaluateExpandFunction(const QString &function, const QString &arguments);
    QString format(const char *format) const;

    QString currentFileName() const;
    QString currentDirectory() const;
    ProFile *currentProFile() const;

    bool evaluateConditionalFunction(const QString &function, const QString &arguments, bool *result);
    bool evaluateFile(const QString &fileName, bool *result);
    bool evaluateFeatureFile(const QString &fileName, bool *result);

    QStringList qmakeFeaturePaths();

    enum { ConditionTrue, ConditionFalse, ConditionElse };
    int m_condition;
    int m_prevCondition;
    bool m_updateCondition;
    bool m_invertNext;
    int m_skipLevel;
    bool m_cumulative;
    bool m_isFirstVariableValue;
    QString m_lastVarName;
    ProVariable::VariableOperator m_variableOperator;
    QString m_origfile;
    QString m_oldPath;                              // To restore the current path to the path
    QStack<ProFile*> m_profileStack;                // To handle 'include(a.pri), so we can track back to 'a.pro' when finished with 'a.pri'

    // we need the following two variables for handling
    // CONFIG = foo bar $$CONFIG
    QHash<QString, QStringList> m_tempValuemap;         // used while evaluating (variable operator value1 value2 ...)
    QHash<const ProFile*, QHash<QString, QStringList> > m_tempFilevaluemap; // used while evaluating (variable operator value1 value2 ...)

    QHash<QString, QStringList> m_valuemap;         // VariableName must be us-ascii, the content however can be non-us-ascii.
    QHash<const ProFile*, QHash<QString, QStringList> > m_filevaluemap; // Variables per include file
    QHash<QString, QString> m_properties;
    QString m_outputDir;

    int m_prevLineNo;                               // Checking whether we're assigning the same TARGET
    ProFile *m_prevProFile;                         // See m_prevLineNo
};

ProFileEvaluator::Private::Private(ProFileEvaluator *q_)
  : q(q_)
{
    m_prevLineNo = 0;
    m_prevProFile = 0;
    m_verbose = true;
    m_block = 0;
    m_commentItem = 0;
    m_syntaxError = 0;
    m_lineNo = 0;
    m_contNextLine = false;
    m_cumulative = true;
    m_updateCondition = false;
    m_condition = ConditionFalse;
    m_invertNext = false;
    m_skipLevel = 0;
    m_isFirstVariableValue = true;
}

bool ProFileEvaluator::Private::read(ProFile *pro)
{
    QFile file(pro->fileName());
    if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
        q->errorMessage(format("%1 not readable.").arg(pro->fileName()));
        return false;
    }

    m_block = 0;
    m_commentItem = 0;
    m_contNextLine = false;
    m_syntaxError = false;
    m_lineNo = 1;
    m_blockstack.clear();
    m_blockstack.push(pro);

    QTextStream ts(&file);
    while (!ts.atEnd()) {
        QString line = ts.readLine();
        if (!parseLine(line)) {
            q->errorMessage(format(".pro parse failure."));
            return false;
        }
        ++m_lineNo;
    }
    return true;
}

bool ProFileEvaluator::Private::parseLine(const QString &line0)
{<commit_msg>amend 3570e978058816745101a69b552ad9d07e349542 for master<commit_after>/****************************************************************************
**
** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
** Contact: Qt Software Information (<EMAIL>)
**
** This file is part of the Qt Linguist of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** No Commercial Usage
** This file contains pre-release code and may not be distributed.
** You may use this file in accordance with the terms and conditions
** contained in the either Technology Preview License Agreement or the
** Beta Release License Agreement.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Nokia gives you certain
** additional rights. These rights are described in the Nokia Qt LGPL
** Exception version 1.0, included in the file LGPL_EXCEPTION.txt in this
** package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
** If you are unsure which license is appropriate for your use, please
** contact the sales department at <EMAIL>.
** $QT_END_LICENSE$
**
****************************************************************************/

#include "profileevaluator.h"
#include "proparserutils.h"
#include "proitems.h"

#include <QtCore/QByteArray>
#include <QtCore/QDateTime>
#include <QtCore/QDebug>
#include <QtCore/QDir>
#include <QtCore/QFile>
#include <QtCore/QFileInfo>
#include <QtCore/QList>
#include <QtCore/QRegExp>
#include <QtCore/QSet>
#include <QtCore/QStack>
#include <QtCore/QString>
#include <QtCore/QStringList>
#include <QtCore/QTextStream>

#ifdef Q_OS_UNIX
#include <unistd.h>
#include <sys/utsname.h>
#elif defined(Q_OS_WIN32)
#include <Windows.h>
#endif
#include <stdio.h>
#include <stdlib.h>

#ifdef Q_OS_WIN32
#define QT_POPEN _popen
#else
#define QT_POPEN popen
#endif

QT_BEGIN_NAMESPACE

///////////////////////////////////////////////////////////////////////
//
// Option
//
///////////////////////////////////////////////////////////////////////

QString
Option::fixString(QString string, uchar flags)
{
    // XXX Ripped out caching, so this will be slow. Should not matter for current uses.

    //fix the environment variables
    if (flags & Option::FixEnvVars) {
        int rep;
        QRegExp reg_variableName(QLatin1String("\\$\\(.*\\)"));
        reg_variableName.setMinimal(true);
        while ((rep = reg_variableName.indexIn(string)) != -1)
            string.replace(rep, reg_variableName.matchedLength(),
                           QString::fromLocal8Bit(qgetenv(string.mid(rep + 2, reg_variableName.matchedLength() - 3).toLatin1().constData()).constData()));
    }

    //canonicalize it (and treat as a path)
    if (flags & Option::FixPathCanonicalize) {
#if 0
        string = QFileInfo(string).canonicalFilePath();
#endif
        string = QDir::cleanPath(string);
    }

    if (string.length() > 2 && string[0].isLetter() && string[1] == QLatin1Char(':'))
        string[0] = string[0].toLower();

    //fix separators
    Q_ASSERT(!((flags & Option::FixPathToLocalSeparators) && (flags & Option::FixPathToTargetSeparators)));
    if (flags & Option::FixPathToLocalSeparators) {
#if defined(Q_OS_WIN32)
        string = string.replace(QLatin1Char('/'), QLatin1Char('\\'));
#else
        string = string.replace(QLatin1Char('\\'), QLatin1Char('/'));
#endif
    } else if (flags & Option::FixPathToTargetSeparators) {
        string = string.replace(QLatin1Char('/'), Option::dir_sep)
                       .replace(QLatin1Char('\\'), Option::dir_sep);
    }

    if ((string.startsWith(QLatin1Char('"')) && string.endsWith(QLatin1Char('"'))) ||
        (string.startsWith(QLatin1Char('\'')) && string.endsWith(QLatin1Char('\''))))
        string = string.mid(1, string.length() - 2);

    return string;
}

///////////////////////////////////////////////////////////////////////
//
// ProFileEvaluator::Private
//
///////////////////////////////////////////////////////////////////////

class ProFileEvaluator::Private : public AbstractProItemVisitor
{
public:
    Private(ProFileEvaluator *q_);

    ProFileEvaluator *q;
    int m_lineNo;                                   // Error reporting
    bool m_verbose;

    /////////////// Reading pro file

    bool read(ProFile *pro);

    ProBlock *currentBlock();
    void updateItem();
    bool parseLine(const QString &line);
    void insertVariable(const QString &line, int *i);
    void insertOperator(const char op);
    void insertComment(const QString &comment);
    void enterScope(bool multiLine);
    void leaveScope();
    void finalizeBlock();

    QStack<ProBlock *> m_blockstack;
    ProBlock *m_block;

    ProItem *m_commentItem;
    QString m_proitem;
    QString m_pendingComment;
    bool m_syntaxError;
    bool m_contNextLine;

    /////////////// Evaluating pro file contents

    // implementation of AbstractProItemVisitor
    bool visitBeginProBlock(ProBlock *block);
    bool visitEndProBlock(ProBlock *block);
    bool visitBeginProVariable(ProVariable *variable);
    bool visitEndProVariable(ProVariable *variable);
    bool visitBeginProFile(ProFile *value);
    bool visitEndProFile(ProFile *value);
    bool visitProValue(ProValue *value);
    bool visitProFunction(ProFunction *function);
    bool visitProOperator(ProOperator *oper);
    bool visitProCondition(ProCondition *condition);

    QStringList valuesDirect(const QString &variableName) const { return m_valuemap[variableName]; }
    QStringList values(const QString &variableName) const;
    QStringList values(const QString &variableName, const ProFile *pro) const;
    QStringList values(const QString &variableName, const QHash<QString, QStringList> &place,
                       const ProFile *pro) const;
    QString propertyValue(const QString &val) const;

    bool isActiveConfig(const QString &config, bool regex = false);
    QStringList expandPattern(const QString &pattern);
    void expandPatternHelper(const QString &relName, const QString &absName,
        QStringList &sources_out);
    QStringList expandVariableReferences(const QString &value);
    QStringList evaluateExpandFunction(const QString &function, const QString &arguments);
    QString format(const char *format) const;

    QString currentFileName() const;
    QString currentDirectory() const;
    ProFile *currentProFile() const;

    bool evaluateConditionalFunction(const QString &function, const QString &arguments, bool *result);
    bool evaluateFile(const QString &fileName, bool *result);
    bool evaluateFeatureFile(const QString &fileName, bool *result);

    QStringList qmakeFeaturePaths();

    enum { ConditionTrue, ConditionFalse, ConditionElse };
    int m_condition;
    int m_prevCondition;
    bool m_updateCondition;
    bool m_invertNext;
    int m_skipLevel;
    bool m_cumulative;
    bool m_isFirstVariableValue;
    QString m_lastVarName;
    ProVariable::VariableOperator m_variableOperator;
    QString m_origfile;
    QString m_oldPath;                              // To restore the current path to the path
    QStack<ProFile*> m_profileStack;                // To handle 'include(a.pri), so we can track back to 'a.pro' when finished with 'a.pri'

    // we need the following two variables for handling
    // CONFIG = foo bar $$CONFIG
    QHash<QString, QStringList> m_tempValuemap;         // used while evaluating (variable operator value1 value2 ...)
    QHash<const ProFile*, QHash<QString, QStringList> > m_tempFilevaluemap; // used while evaluating (variable operator value1 value2 ...)

    QHash<QString, QStringList> m_valuemap;         // VariableName must be us-ascii, the content however can be non-us-ascii.
    QHash<const ProFile*, QHash<QString, QStringList> > m_filevaluemap; // Variables per include file
    QHash<QString, QString> m_properties;
    QString m_outputDir;

    int m_prevLineNo;                               // Checking whether we're assigning the same TARGET
    ProFile *m_prevProFile;                         // See m_prevLineNo
};

ProFileEvaluator::Private::Private(ProFileEvaluator *q_)
  : q(q_)
{
    // Global parser state
    m_prevLineNo = 0;
    m_prevProFile = 0;

    // Configuration, more or less
    m_verbose = true;
    m_cumulative = true;

    // Evaluator state
    m_updateCondition = false;
    m_condition = ConditionFalse;
    m_invertNext = false;
    m_skipLevel = 0;
    m_isFirstVariableValue = true;
}

bool ProFileEvaluator::Private::read(ProFile *pro)
{
    QFile file(pro->fileName());
    if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
        q->errorMessage(format("%1 not readable.").arg(pro->fileName()));
        return false;
    }

    // Parser state
    m_block = 0;
    m_commentItem = 0;
    m_contNextLine = false;
    m_syntaxError = false;
    m_lineNo = 1;
    m_blockstack.clear();
    m_blockstack.push(pro);

    QTextStream ts(&file);
    while (!ts.atEnd()) {
        QString line = ts.readLine();
        if (!parseLine(line)) {
            q->errorMessage(format(".pro parse failure."));
            return false;
        }
        ++m_lineNo;
    }
    return true;
}

bool ProFileEvaluator::Private::parseLine(const QString &line0)
{
{'fraction_non_alphanumeric': 0.149197592778335, 'fraction_numerical': 0.007773319959879639, 'mean_word_length': 3.1857682619647356, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1549825', 'n_tokens_mistral': 5809, 'n_tokens_neox': 5596, 'n_words': 1634}