<commit_before>/****************************************************************************
**
** Copyright (C) 2016 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qbs.
**
** $QT_BEGIN_LICENSE:LGPL$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 3 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL3 included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 3 requirements
** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 2.0 or (at your option) the GNU General
** Public license version 3 or any later version approved by the KDE Free
** Qt Foundation. The licenses are as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-2.0.html and
** https://www.gnu.org/licenses/gpl-3.0.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/

#include "moduleloader.h"

#include "builtindeclarations.h"
#include "evaluator.h"
#include "filecontext.h"
#include "item.h"
#include "itemreader.h"
#include "language.h"
#include "modulemerger.h"
#include "qualifiedid.h"
#include "scriptengine.h"
#include "value.h"

#include <language/language.h>
#include <language/scriptengine.h>
#include <logging/logger.h>
#include <logging/translator.h>
#include <tools/error.h>
#include <tools/fileinfo.h>
#include <tools/preferences.h>
#include <tools/profile.h>
#include <tools/progressobserver.h>
#include <tools/qbsassert.h>
#include <tools/scripttools.h>
#include <tools/settings.h>

#include <QDebug>
#include <QDir>
#include <QDirIterator>
#include <QPair>

#include <algorithm>

namespace qbs {
namespace Internal {

class ModuleLoader::ItemModuleList : public QList<Item::Module> {};

const QString moduleSearchSubDir = QLatin1String("modules");

class ModuleLoader::ProductSortByDependencies
{
public:
    ProductSortByDependencies(TopLevelProjectContext &tlp) : m_tlp(tlp)
    {
    }

    void apply()
    {
        QHash<QString, ProductContext *> productsMap;
        QList<ProductContext *> allProducts;
        for (auto projIt = m_tlp.projects.begin(); projIt != m_tlp.projects.end(); ++projIt) {
            QVector<ProductContext> &products = (*projIt)->products;
            for (auto prodIt = products.begin(); prodIt != products.end(); ++prodIt) {
                allProducts << prodIt;
                productsMap.insert(prodIt->name, prodIt);
            }
        }
        QSet<ProductContext *> allDependencies;
        foreach (auto productContext, allProducts) {
            auto &productDependencies = m_dependencyMap[productContext];
            foreach (const auto &dep, productContext->info.usedProducts) {
                if (!dep.productTypes.isEmpty())
                    continue;
                QBS_CHECK(!dep.name.isEmpty());
                ProductContext * const depProduct = productsMap.value(dep.name);
                QBS_CHECK(depProduct);
                productDependencies << depProduct;
                allDependencies << depProduct;
            }
        }
        QSet<ProductContext *> rootProducts = allProducts.toSet() - allDependencies;
        foreach (ProductContext * const rootProduct, rootProducts)
            traverse(rootProduct);
        if (m_sortedProducts.count() < allProducts.count()) {
            foreach (auto * const product, allProducts) {
                QList<ModuleLoader::ProductContext *> path;
                findCycle(product, path);
            }
        }
        QBS_CHECK(m_sortedProducts.count() == allProducts.count());
    }

    // No product at position i has dependencies to a product at position j > i.
    QList<ProductContext *> sortedProducts()
    {
        return m_sortedProducts;
    }

private:
    void traverse(ModuleLoader::ProductContext *product)
    {
        if (m_seenProducts.contains(product))
            return;
        m_seenProducts << product;
        foreach (auto dependency, m_dependencyMap.value(product))
            traverse(dependency);
        m_sortedProducts << product;
    }

    void findCycle(ModuleLoader::ProductContext *product,
                   QList<ModuleLoader::ProductContext *> &path)
    {
        if (path.contains(product)) {
            ErrorInfo error(Tr::tr("Cyclic dependencies detected."));
            foreach (const auto * const p, path)
                error.append(p->name, p->item->location());
            error.append(product->name, product->item->location());
            throw error;
        }
        path << product;
        foreach (auto * const dep, m_dependencyMap.value(product))
            findCycle(dep, path);
        path.removeLast();
    }

    TopLevelProjectContext &m_tlp;
    QHash<ProductContext *, QVector<ProductContext *>> m_dependencyMap;
    QSet<ProductContext *> m_seenProducts;
    QList<ProductContext *> m_sortedProducts;
};

class SearchPathsManager {
public:
    SearchPathsManager(ItemReader *itemReader, const QStringList &extraSearchPaths)
        : m_itemReader(itemReader)
    {
        m_itemReader->pushExtraSearchPaths(extraSearchPaths);
    }
    ~SearchPathsManager() { m_itemReader->popExtraSearchPaths(); }

private:
    ItemReader * const m_itemReader;
};

ModuleLoader::ModuleLoader(ScriptEngine *engine,
                           Logger &logger)
    : m_engine(engine)
    , m_pool(0)
    , m_logger(logger)
    , m_progressObserver(0)
    , m_reader(new ItemReader(logger))
    , m_evaluator(new Evaluator(engine, logger))
{
}

ModuleLoader::~ModuleLoader()
{
    delete m_evaluator;
    delete m_reader;
}

void ModuleLoader::setProgressObserver(ProgressObserver *progressObserver)
{
    m_progressObserver = progressObserver;
}

static void addExtraModuleSearchPath(QStringList &list, const QString &searchPath)
{
    list += FileInfo::resolvePath(searchPath, moduleSearchSubDir);
}

void ModuleLoader::setSearchPaths(const QStringList &searchPaths)
{
    m_reader->setSearchPaths(searchPaths);

    m_moduleDirListCache.clear();
    m_moduleSearchPaths.clear();
    foreach (const QString &path, searchPaths)
        addExtraModuleSearchPath(m_moduleSearchPaths, path);

    if (m_logger.traceEnabled()) {
        m_logger.qbsTrace() << "[MODLDR] module search paths:";
        foreach (const QString &path, m_moduleSearchPaths)
            m_logger.qbsTrace() << "    " << path;
    }
}

void ModuleLoader::setOldProbes(const QHash<QString, QList<ProbeConstPtr>> &oldProbes)
{
    m_oldProbes = oldProbes;
}

ModuleLoaderResult ModuleLoader::load(const SetupProjectParameters &parameters)
{
    if (m_logger.traceEnabled())
        m_logger.qbsTrace() << "[MODLDR] load" << parameters.projectFilePath();
    m_parameters = parameters;
    m_productModuleCache.clear();
    m_modulePrototypeItemCache.clear();
    m_disabledItems.clear();
    m_reader->clearExtraSearchPathsStack();

    ModuleLoaderResult result;
    m_pool = result.itemPool.data();
    m_reader->setPool(m_pool);

    const QStringList topLevelSearchPaths = parameters.finalBuildConfigurationTree()
            .value(QLatin1String("project")).toMap()
            .value(QLatin1String("qbsSearchPaths")).toStringList();
    Item *root;
    {
        SearchPathsManager searchPathsManager(m_reader, topLevelSearchPaths);
        root = m_reader->readFile(parameters.projectFilePath());
        if (!root)
            return ModuleLoaderResult();
    }

    root = wrapInProjectIfNecessary(root);

    const QString buildDirectory = TopLevelProject::deriveBuildDirectory(parameters.buildRoot(),
            TopLevelProject::deriveId(parameters.finalBuildConfigurationTree()));
    root->setProperty(QLatin1String("sourceDirectory"),
                      VariantValue::create(QFileInfo(root->file()->filePath()).absolutePath()));
    root->setProperty(QLatin1String("buildDirectory"), VariantValue::create(buildDirectory));
    root->setProperty(QLatin1String("profile"),
                      VariantValue::create(m_parameters.topLevelProfile()));
    handleTopLevelProject(&result, root, buildDirectory,
                  QSet<QString>() << QDir::cleanPath(parameters.projectFilePath()));
    result.root = root;
    result.qbsFiles = m_reader->filesRead();
    return result;
}

static void handlePropertyError(const ErrorInfo &error, const SetupProjectParameters &params,
                                Logger &logger)
{
    if (params.propertyCheckingMode() == ErrorHandlingMode::Strict)
        throw error;
    logger.printWarning(error);
}

class PropertyDeclarationCheck : public ValueHandler
{
    const QSet<Item *> &m_disabledItems;
    Item *m_parentItem;
    QString m_currentName;
    SetupProjectParameters m_params;
    Logger &m_logger;
public:
    PropertyDeclarationCheck(const QSet<Item *> &disabledItems,
                             const SetupProjectParameters &params, Logger &logger)
        : m_disabledItems(disabledItems)
        , m_parentItem(0)
        , m_params(params)
        , m_logger(logger)
    {
    }

    void operator()(Item *item)
    {
        handleItem(item);
    }

private:
    void handle(JSSourceValue *value)
    {
        if (!value->createdByPropertiesBlock()) {
            const ErrorInfo error(Tr::tr("Property '%1' is not declared.")
                                  .arg(m_currentName), value->location());
            handlePropertyError(error, m_params, m_logger);
        }
    }

    void handle(ItemValue *value)
    {
        if (value->item()->type() != ItemType::ModuleInstance
                && value->item()->type() != ItemType::ModulePrefix
                && m_parentItem->file()
                && (!m_parentItem->file()->idScope()
                    || !m_parentItem->file()->idScope()->hasProperty(m_currentName))
                && !value->createdByPropertiesBlock()) {
            const ErrorInfo error(Tr::tr("Item '%1' is not declared. "
                                         "Did you forget to add a Depends item?").arg(m_currentName),
                                  value->location().isValid() ? value->location()
                                                              : m_parentItem->location());
            handlePropertyError(error, m_params, m_logger);
        } else {
            handleItem(value->item());
        }
    }

    void handleItem(Item *item)
    {
        if (m_disabledItems.contains(item)
                // TODO: We never checked module prototypes, apparently. Should we?
                // It's currently not possible because of e.g. things like "cpp.staticLibraries"
                // inside Artifact items...
                || item->type() == ItemType::Module

                // The Properties child of a SubProject item is not a regular item.
                || item->type() == ItemType::PropertiesInSubProject) {
            return;
        }

        Item *oldParentItem = m_parentItem;
        m_parentItem = item;
        for (Item::PropertyMap::const_iterator it = item->properties().constBegin();
                it != item->properties().constEnd(); ++it) {
            const PropertyDeclaration decl = item->propertyDeclaration(it.key());
            if (decl.isValid()) {
                if (!decl.isDeprecated())
                    continue;
                const DeprecationInfo &di = decl.deprecationInfo();
                QString message;
                bool warningOnly;
                if (di.removalVersion() <= Version::qbsVersion()) {
                    message = Tr::tr("The property '%1' can no longer be used. "
                                     "It was removed in Qbs %2.")
                            .arg(decl.name(), di.removalVersion().toString());<commit_msg>ModuleLoader: Remove redundant property checks<commit_after>/****************************************************************************
**
** Copyright (C) 2016 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qbs.
**
** $QT_BEGIN_LICENSE:LGPL$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 3 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL3 included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 3 requirements
** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 2.0 or (at your option) the GNU General
** Public license version 3 or any later version approved by the KDE Free
** Qt Foundation. The licenses are as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-2.0.html and
** https://www.gnu.org/licenses/gpl-3.0.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/

#include "moduleloader.h"

#include "builtindeclarations.h"
#include "evaluator.h"
#include "filecontext.h"
#include "item.h"
#include "itemreader.h"
#include "language.h"
#include "modulemerger.h"
#include "qualifiedid.h"
#include "scriptengine.h"
#include "value.h"

#include <language/language.h>
#include <language/scriptengine.h>
#include <logging/logger.h>
#include <logging/translator.h>
#include <tools/error.h>
#include <tools/fileinfo.h>
#include <tools/preferences.h>
#include <tools/profile.h>
#include <tools/progressobserver.h>
#include <tools/qbsassert.h>
#include <tools/scripttools.h>
#include <tools/settings.h>

#include <QDebug>
#include <QDir>
#include <QDirIterator>
#include <QPair>

#include <algorithm>

namespace qbs {
namespace Internal {

class ModuleLoader::ItemModuleList : public QList<Item::Module> {};

const QString moduleSearchSubDir = QLatin1String("modules");

class ModuleLoader::ProductSortByDependencies
{
public:
    ProductSortByDependencies(TopLevelProjectContext &tlp) : m_tlp(tlp)
    {
    }

    void apply()
    {
        QHash<QString, ProductContext *> productsMap;
        QList<ProductContext *> allProducts;
        for (auto projIt = m_tlp.projects.begin(); projIt != m_tlp.projects.end(); ++projIt) {
            QVector<ProductContext> &products = (*projIt)->products;
            for (auto prodIt = products.begin(); prodIt != products.end(); ++prodIt) {
                allProducts << prodIt;
                productsMap.insert(prodIt->name, prodIt);
            }
        }
        QSet<ProductContext *> allDependencies;
        foreach (auto productContext, allProducts) {
            auto &productDependencies = m_dependencyMap[productContext];
            foreach (const auto &dep, productContext->info.usedProducts) {
                if (!dep.productTypes.isEmpty())
                    continue;
                QBS_CHECK(!dep.name.isEmpty());
                ProductContext * const depProduct = productsMap.value(dep.name);
                QBS_CHECK(depProduct);
                productDependencies << depProduct;
                allDependencies << depProduct;
            }
        }
        QSet<ProductContext *> rootProducts = allProducts.toSet() - allDependencies;
        foreach (ProductContext * const rootProduct, rootProducts)
            traverse(rootProduct);
        if (m_sortedProducts.count() < allProducts.count()) {
            foreach (auto * const product, allProducts) {
                QList<ModuleLoader::ProductContext *> path;
                findCycle(product, path);
            }
        }
        QBS_CHECK(m_sortedProducts.count() == allProducts.count());
    }

    // No product at position i has dependencies to a product at position j > i.
    QList<ProductContext *> sortedProducts()
    {
        return m_sortedProducts;
    }

private:
    void traverse(ModuleLoader::ProductContext *product)
    {
        if (m_seenProducts.contains(product))
            return;
        m_seenProducts << product;
        foreach (auto dependency, m_dependencyMap.value(product))
            traverse(dependency);
        m_sortedProducts << product;
    }

    void findCycle(ModuleLoader::ProductContext *product,
                   QList<ModuleLoader::ProductContext *> &path)
    {
        if (path.contains(product)) {
            ErrorInfo error(Tr::tr("Cyclic dependencies detected."));
            foreach (const auto * const p, path)
                error.append(p->name, p->item->location());
            error.append(product->name, product->item->location());
            throw error;
        }
        path << product;
        foreach (auto * const dep, m_dependencyMap.value(product))
            findCycle(dep, path);
        path.removeLast();
    }

    TopLevelProjectContext &m_tlp;
    QHash<ProductContext *, QVector<ProductContext *>> m_dependencyMap;
    QSet<ProductContext *> m_seenProducts;
    QList<ProductContext *> m_sortedProducts;
};

class SearchPathsManager {
public:
    SearchPathsManager(ItemReader *itemReader, const QStringList &extraSearchPaths)
        : m_itemReader(itemReader)
    {
        m_itemReader->pushExtraSearchPaths(extraSearchPaths);
    }
    ~SearchPathsManager() { m_itemReader->popExtraSearchPaths(); }

private:
    ItemReader * const m_itemReader;
};

ModuleLoader::ModuleLoader(ScriptEngine *engine,
                           Logger &logger)
    : m_engine(engine)
    , m_pool(0)
    , m_logger(logger)
    , m_progressObserver(0)
    , m_reader(new ItemReader(logger))
    , m_evaluator(new Evaluator(engine, logger))
{
}

ModuleLoader::~ModuleLoader()
{
    delete m_evaluator;
    delete m_reader;
}

void ModuleLoader::setProgressObserver(ProgressObserver *progressObserver)
{
    m_progressObserver = progressObserver;
}

static void addExtraModuleSearchPath(QStringList &list, const QString &searchPath)
{
    list += FileInfo::resolvePath(searchPath, moduleSearchSubDir);
}

void ModuleLoader::setSearchPaths(const QStringList &searchPaths)
{
    m_reader->setSearchPaths(searchPaths);

    m_moduleDirListCache.clear();
    m_moduleSearchPaths.clear();
    foreach (const QString &path, searchPaths)
        addExtraModuleSearchPath(m_moduleSearchPaths, path);

    if (m_logger.traceEnabled()) {
        m_logger.qbsTrace() << "[MODLDR] module search paths:";
        foreach (const QString &path, m_moduleSearchPaths)
            m_logger.qbsTrace() << "    " << path;
    }
}

void ModuleLoader::setOldProbes(const QHash<QString, QList<ProbeConstPtr>> &oldProbes)
{
    m_oldProbes = oldProbes;
}

ModuleLoaderResult ModuleLoader::load(const SetupProjectParameters &parameters)
{
    if (m_logger.traceEnabled())
        m_logger.qbsTrace() << "[MODLDR] load" << parameters.projectFilePath();
    m_parameters = parameters;
    m_productModuleCache.clear();
    m_modulePrototypeItemCache.clear();
    m_disabledItems.clear();
    m_reader->clearExtraSearchPathsStack();

    ModuleLoaderResult result;
    m_pool = result.itemPool.data();
    m_reader->setPool(m_pool);

    const QStringList topLevelSearchPaths = parameters.finalBuildConfigurationTree()
            .value(QLatin1String("project")).toMap()
            .value(QLatin1String("qbsSearchPaths")).toStringList();
    Item *root;
    {
        SearchPathsManager searchPathsManager(m_reader, topLevelSearchPaths);
        root = m_reader->readFile(parameters.projectFilePath());
        if (!root)
            return ModuleLoaderResult();
    }

    root = wrapInProjectIfNecessary(root);

    const QString buildDirectory = TopLevelProject::deriveBuildDirectory(parameters.buildRoot(),
            TopLevelProject::deriveId(parameters.finalBuildConfigurationTree()));
    root->setProperty(QLatin1String("sourceDirectory"),
                      VariantValue::create(QFileInfo(root->file()->filePath()).absolutePath()));
    root->setProperty(QLatin1String("buildDirectory"), VariantValue::create(buildDirectory));
    root->setProperty(QLatin1String("profile"),
                      VariantValue::create(m_parameters.topLevelProfile()));
    handleTopLevelProject(&result, root, buildDirectory,
                  QSet<QString>() << QDir::cleanPath(parameters.projectFilePath()));
    result.root = root;
    result.qbsFiles = m_reader->filesRead();
    return result;
}

static void handlePropertyError(const ErrorInfo &error, const SetupProjectParameters &params,
                                Logger &logger)
{
    if (params.propertyCheckingMode() == ErrorHandlingMode::Strict)
        throw error;
    logger.printWarning(error);
}

class PropertyDeclarationCheck : public ValueHandler
{
    const QSet<Item *> &m_disabledItems;
    QSet<Item *> m_handledItems;
    Item *m_parentItem;
    QString m_currentName;
    SetupProjectParameters m_params;
    Logger &m_logger;
public:
    PropertyDeclarationCheck(const QSet<Item *> &disabledItems,
                             const SetupProjectParameters &params, Logger &logger)
        : m_disabledItems(disabledItems)
        , m_parentItem(0)
        , m_params(params)
        , m_logger(logger)
    {
    }

    void operator()(Item *item)
    {
        handleItem(item);
    }

private:
    void handle(JSSourceValue *value)
    {
        if (!value->createdByPropertiesBlock()) {
            const ErrorInfo error(Tr::tr("Property '%1' is not declared.")
                                  .arg(m_currentName), value->location());
            handlePropertyError(error, m_params, m_logger);
        }
    }

    void handle(ItemValue *value)
    {
        if (value->item()->type() != ItemType::ModuleInstance
                && value->item()->type() != ItemType::ModulePrefix
                && m_parentItem->file()
                && (!m_parentItem->file()->idScope()
                    || !m_parentItem->file()->idScope()->hasProperty(m_currentName))
                && !value->createdByPropertiesBlock()) {
            const ErrorInfo error(Tr::tr("Item '%1' is not declared. "
                                         "Did you forget to add a Depends item?").arg(m_currentName),
                                  value->location().isValid() ? value->location()
                                                              : m_parentItem->location());
            handlePropertyError(error, m_params, m_logger);
        } else {
            handleItem(value->item());
        }
    }

    void handleItem(Item *item)
    {
        if (m_handledItems.contains(item))
            return;
        m_handledItems.insert(item);
        if (m_disabledItems.contains(item)
                // TODO: We never checked module prototypes, apparently. Should we?
                // It's currently not possible because of e.g. things like "cpp.staticLibraries"
                // inside Artifact items...
                || item->type() == ItemType::Module

                // The Properties child of a SubProject item is not a regular item.
                || item->type() == ItemType::PropertiesInSubProject) {
            return;
        }

        Item *oldParentItem = m_parentItem;
        m_parentItem = item;
        for (Item::PropertyMap::const_iterator it = item->properties().constBegin();
                it != item->properties().constEnd(); ++it) {
            const PropertyDeclaration decl = item->propertyDeclaration(it.key());
            if (decl.isValid()) {
                if (!decl.isDeprecated())
                    continue;
                const DeprecationInfo &di = decl.deprecationInfo();
                QString message;
                bool warningOnly;
                if (di.removalVersion() <= Version::qbsVersion()) {
                    message = Tr::tr("The property '%1' can no longer be used. "
                                     "It was removed in Qbs %2.")
                            .arg(decl.name(), di.removalVersion().toString());
{'fraction_non_alphanumeric': 0.12331081081081081, 'fraction_numerical': 0.002435575109993715, 'mean_word_length': 2.8735544735240413, 'pattern_counts': {'":': 0, '<': 126, '<?xml version=': 0, '>': 162, 'https://': 12, 'lorem ipsum': 0, 'www.': 12, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4217203', 'n_tokens_mistral': 6965, 'n_tokens_neox': 6732, 'n_words': 1603}