<commit_before>/******************************************************************************
 Copyright (c) 2016,17 <NAME> (<EMAIL>)

 This library is free software; you can redistribute it and/or modify it under
 the terms of the GNU Lesser General Public License as published by the Free
 Software Foundation; either version 2.1 of the License, or (at your option)
 any later version.

 This library is distributed in the hope that it will be useful, but WITHOUT
 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
 for more details.

 You should have received a copy of the GNU Lesser General Public License along
 with this library; if not, write to the Free Software Foundation, Inc., 51
 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
 *****************************************************************************/

#include "Driver.h"
#include "AST.h"
#include "ASTDumper.h"
#include "ASTNormalizer.h"
#include "BaseTester.h"
#include "Bind.h"
#include "CompilerFacade.h"
#include "ConstraintGenerator.h"
#include "ConstraintWriter.h"
#include "DeclarationInterceptor.h"
#include "Debug.h"
#include "DiagnosticCollector.h"
#include "DomainLattice.h"
#include "IO.h"
#include "Literals.h"
#include "Observer.h"
#include "ProgramValidator.h"
#include "Symbols.h"
#include "StdLibIndex.h"

#include "cxxopts.hpp"

#include <iostream>
#include <sstream>
#include <cstring>

using namespace psyche;
using namespace CPlusPlus;
using namespace std::placeholders;

namespace psyche {
    extern bool debugEnabled;
}

constexpr int Driver::OK;
constexpr int Driver::ParsingError;
constexpr int Driver::InvalidSyntax;
constexpr int Driver::UnavailableAstError;
constexpr int Driver::UnresolvedAmbiguity;
constexpr int Driver::UnspecifiedInputFile;
constexpr int Driver::UnknownCommandLineOption;
constexpr int Driver::InvalidCommandLineValue;

Driver::Driver(const Factory& factory)
    : factory_(factory)
    , global_(nullptr)
{}

int Driver::process(int argc, char *argv[])
{
    ExecutionOptions exeOpts;
    std::string in;
    cxxopts::Options cmdOpts(argv[0], "PsycheC constraint generator");

    try {
        cmdOpts.positional_help("file");

        cmdOpts.add_options()
            ("h,help", "Print help")
            ("o,output", "Specify output file",
                cxxopts::value<std::string>()->default_value("a.cstr"))
            ("d,debug", "Enable debugging messages",
                cxxopts::value<bool>(debugEnabled))
            ("c,constraints", "Display generated constraints")
            ("dump-ast", "Dump AST in .dot format")
            ("s,stats", "Display AST disambiguation and constraint-generation stats")
            ("t,test-disambiguation", "Run AST disambiguation tests")
            ("no-heuristic", "Disable heuristics on unresolved AST ambiguities")
            ("no-typedef", "Forbid typedef and struct/union declarations")
            ("match-stdlib", "Match stdlib names: ignore, approx, strict",
                cxxopts::value<std::string>()->default_value("ignore"))
            ("stdlib-inc", "Required #includes from stdlib",
                cxxopts::value<std::string>()->default_value("a.inc"))
            ("CC", "Specify host C compiler",
                cxxopts::value<std::string>()->default_value("gcc"))
            ("cc-D", "Predefine a macro",
                cxxopts::value<std::vector<std::string>>())
            ("cc-U", "Undefine a macro",
                cxxopts::value<std::vector<std::string>>())
            ("positional", "Positional arguments",
                cxxopts::value<std::vector<std::string>>());

        cmdOpts.parse_positional(std::vector<std::string>{"file", "positional"});
        cmdOpts.parse(argc, argv);
    } catch (const cxxopts::OptionException& e) {
        std::cerr << kPsychePrefix << e.what() << std::endl;
        return UnknownCommandLineOption;
    }

    if (cmdOpts.count("help")) {
        std::cout << cmdOpts.help({"", "Group"}) << std::endl;
        return OK;
    }

    if (cmdOpts.count("test-disambiguation")) {
        try {
            BaseTester::runSuite();
        } catch (...) {
            std::cout << "\nYou BROKE stuff! Take a look at it!" << std::endl;
        }
        return OK;
    }

    if (!cmdOpts.count("positional")) {
        std::cerr << kPsychePrefix << "unspecified input file" << std::endl;
        return UnspecifiedInputFile;
    }

    // The input file name is the single positional argument.
    if (cmdOpts.count("positional")) {
        auto& v = cmdOpts["positional"].as<std::vector<std::string>>();
        in = v[0];
    }

    const auto& matchLib = cmdOpts["match-stdlib"].as<std::string>();
    if (matchLib == "ignore")
        exeOpts.flag_.stdlibMode = static_cast<uint32_t>(StdLibMode::Ignore);
    else if (matchLib == "approx")
        exeOpts.flag_.stdlibMode = static_cast<uint32_t>(StdLibMode::Approx);
    else if (matchLib == "strict")
        exeOpts.flag_.stdlibMode = static_cast<uint32_t>(StdLibMode::Strict);
    else {
        std::cerr << kPsychePrefix << "invalid command line value" << std::endl;
        return InvalidCommandLineValue;
    }

    exeOpts.flag_.dumpAst = cmdOpts.count("dump-ast");
    exeOpts.flag_.displayConstraints = cmdOpts.count("constraints");
    exeOpts.flag_.displayStats = cmdOpts.count("stats");
    exeOpts.flag_.noHeuristics = cmdOpts.count("no-heuristic");
    exeOpts.flag_.noTypedef = cmdOpts.count("no-typedef");
    exeOpts.flag_.handleGNUerrorFunc_ = true; // TODO: POSIX stuff?
    exeOpts.nativeCC_ = cmdOpts["CC"].as<std::string>();
    exeOpts.defs_ = cmdOpts["cc-D"].as<std::vector<std::string>>();
    exeOpts.undefs_ = cmdOpts["cc-U"].as<std::vector<std::string>>();

    const std::string& source = readFile(in);

    const auto code = process(in, source, exeOpts);
    switch (code) {
    case OK:
        if (!constraints_.empty()) {
            writeFile(constraints_, cmdOpts["output"].as<std::string>());
            if (cmdOpts.count("stdlib-inc") && !includes_.empty())
                writeFile(includes_, cmdOpts["stdlib-inc"].as<std::string>());
        }
        break;

    case ParsingError:
        std::cerr << kPsychePrefix << "parsing (internal) error" << std::endl;
        break;

    case UnavailableAstError:
        std::cerr << kPsychePrefix << "unavailable AST (internal) error" << in << std::endl;
        break;

    default:
        break;
    }

    return code;
}

int Driver::process(const std::string& unitName,
                    const std::string& source,
                    const ExecutionOptions& flags)
{
    configure(flags);

    StringLiteral name(unitName.c_str(), unitName.length());
    unit_.reset(new TranslationUnit(&control_, &name));
    unit_->setDialect(specifiedDialect(flags));

    static DiagnosticCollector collector;
    control_.setDiagnosticCollector(&collector);

    return parse(source, true);
}

TranslationUnit *Driver::tu() const
{
    return unit_.get();
}

Dialect Driver::specifiedDialect(const ExecutionOptions&)
{
    Dialect dialect;
    dialect.c99 = 1;
    dialect.gnuKeywordExt = 0;
    dialect.nullptrOnNULL = 1;

    return dialect;
}

void Driver::configure(const ExecutionOptions& flags)<commit_msg>Enable gnu extended keywords<commit_after>/******************************************************************************
 Copyright (c) 2016,17 <NAME> (<EMAIL>)

 This library is free software; you can redistribute it and/or modify it under
 the terms of the GNU Lesser General Public License as published by the Free
 Software Foundation; either version 2.1 of the License, or (at your option)
 any later version.

 This library is distributed in the hope that it will be useful, but WITHOUT
 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
 for more details.

 You should have received a copy of the GNU Lesser General Public License along
 with this library; if not, write to the Free Software Foundation, Inc., 51
 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
 *****************************************************************************/

#include "Driver.h"
#include "AST.h"
#include "ASTDumper.h"
#include "ASTNormalizer.h"
#include "BaseTester.h"
#include "Bind.h"
#include "CompilerFacade.h"
#include "ConstraintGenerator.h"
#include "ConstraintWriter.h"
#include "DeclarationInterceptor.h"
#include "Debug.h"
#include "DiagnosticCollector.h"
#include "DomainLattice.h"
#include "IO.h"
#include "Literals.h"
#include "Observer.h"
#include "ProgramValidator.h"
#include "Symbols.h"
#include "StdLibIndex.h"

#include "cxxopts.hpp"

#include <iostream>
#include <sstream>
#include <cstring>

using namespace psyche;
using namespace CPlusPlus;
using namespace std::placeholders;

namespace psyche {
    extern bool debugEnabled;
}

constexpr int Driver::OK;
constexpr int Driver::ParsingError;
constexpr int Driver::InvalidSyntax;
constexpr int Driver::UnavailableAstError;
constexpr int Driver::UnresolvedAmbiguity;
constexpr int Driver::UnspecifiedInputFile;
constexpr int Driver::UnknownCommandLineOption;
constexpr int Driver::InvalidCommandLineValue;

Driver::Driver(const Factory& factory)
    : factory_(factory)
    , global_(nullptr)
{}

int Driver::process(int argc, char *argv[])
{
    ExecutionOptions exeOpts;
    std::string in;
    cxxopts::Options cmdOpts(argv[0], "PsycheC constraint generator");

    try {
        cmdOpts.positional_help("file");

        cmdOpts.add_options()
            ("h,help", "Print help")
            ("o,output", "Specify output file",
                cxxopts::value<std::string>()->default_value("a.cstr"))
            ("d,debug", "Enable debugging messages",
                cxxopts::value<bool>(debugEnabled))
            ("c,constraints", "Display generated constraints")
            ("dump-ast", "Dump AST in .dot format")
            ("s,stats", "Display AST disambiguation and constraint-generation stats")
            ("t,test-disambiguation", "Run AST disambiguation tests")
            ("no-heuristic", "Disable heuristics on unresolved AST ambiguities")
            ("no-typedef", "Forbid typedef and struct/union declarations")
            ("match-stdlib", "Match stdlib names: ignore, approx, strict",
                cxxopts::value<std::string>()->default_value("ignore"))
            ("stdlib-inc", "Required #includes from stdlib",
                cxxopts::value<std::string>()->default_value("a.inc"))
            ("CC", "Specify host C compiler",
                cxxopts::value<std::string>()->default_value("gcc"))
            ("cc-D", "Predefine a macro",
                cxxopts::value<std::vector<std::string>>())
            ("cc-U", "Undefine a macro",
                cxxopts::value<std::vector<std::string>>())
            ("positional", "Positional arguments",
                cxxopts::value<std::vector<std::string>>());

        cmdOpts.parse_positional(std::vector<std::string>{"file", "positional"});
        cmdOpts.parse(argc, argv);
    } catch (const cxxopts::OptionException& e) {
        std::cerr << kPsychePrefix << e.what() << std::endl;
        return UnknownCommandLineOption;
    }

    if (cmdOpts.count("help")) {
        std::cout << cmdOpts.help({"", "Group"}) << std::endl;
        return OK;
    }

    if (cmdOpts.count("test-disambiguation")) {
        try {
            BaseTester::runSuite();
        } catch (...) {
            std::cout << "\nYou BROKE stuff! Take a look at it!" << std::endl;
        }
        return OK;
    }

    if (!cmdOpts.count("positional")) {
        std::cerr << kPsychePrefix << "unspecified input file" << std::endl;
        return UnspecifiedInputFile;
    }

    // The input file name is the single positional argument.
    if (cmdOpts.count("positional")) {
        auto& v = cmdOpts["positional"].as<std::vector<std::string>>();
        in = v[0];
    }

    const auto& matchLib = cmdOpts["match-stdlib"].as<std::string>();
    if (matchLib == "ignore")
        exeOpts.flag_.stdlibMode = static_cast<uint32_t>(StdLibMode::Ignore);
    else if (matchLib == "approx")
        exeOpts.flag_.stdlibMode = static_cast<uint32_t>(StdLibMode::Approx);
    else if (matchLib == "strict")
        exeOpts.flag_.stdlibMode = static_cast<uint32_t>(StdLibMode::Strict);
    else {
        std::cerr << kPsychePrefix << "invalid command line value" << std::endl;
        return InvalidCommandLineValue;
    }

    exeOpts.flag_.dumpAst = cmdOpts.count("dump-ast");
    exeOpts.flag_.displayConstraints = cmdOpts.count("constraints");
    exeOpts.flag_.displayStats = cmdOpts.count("stats");
    exeOpts.flag_.noHeuristics = cmdOpts.count("no-heuristic");
    exeOpts.flag_.noTypedef = cmdOpts.count("no-typedef");
    exeOpts.flag_.handleGNUerrorFunc_ = true; // TODO: POSIX stuff?
    exeOpts.nativeCC_ = cmdOpts["CC"].as<std::string>();
    exeOpts.defs_ = cmdOpts["cc-D"].as<std::vector<std::string>>();
    exeOpts.undefs_ = cmdOpts["cc-U"].as<std::vector<std::string>>();

    const std::string& source = readFile(in);

    const auto code = process(in, source, exeOpts);
    switch (code) {
    case OK:
        if (!constraints_.empty()) {
            writeFile(constraints_, cmdOpts["output"].as<std::string>());
            if (cmdOpts.count("stdlib-inc") && !includes_.empty())
                writeFile(includes_, cmdOpts["stdlib-inc"].as<std::string>());
        }
        break;

    case ParsingError:
        std::cerr << kPsychePrefix << "parsing (internal) error" << std::endl;
        break;

    case UnavailableAstError:
        std::cerr << kPsychePrefix << "unavailable AST (internal) error" << in << std::endl;
        break;

    default:
        break;
    }

    return code;
}

int Driver::process(const std::string& unitName,
                    const std::string& source,
                    const ExecutionOptions& flags)
{
    configure(flags);

    StringLiteral name(unitName.c_str(), unitName.length());
    unit_.reset(new TranslationUnit(&control_, &name));
    unit_->setDialect(specifiedDialect(flags));

    static DiagnosticCollector collector;
    control_.setDiagnosticCollector(&collector);

    return parse(source, true);
}

TranslationUnit *Driver::tu() const
{
    return unit_.get();
}

Dialect Driver::specifiedDialect(const ExecutionOptions&)
{
    Dialect dialect;
    dialect.c99 = 1;
    dialect.gnuKeywordExt = 1;
    dialect.nullptrOnNULL = 1;

    return dialect;
}

void Driver::configure(const ExecutionOptions& flags)
{'fraction_non_alphanumeric': 0.15726495726495726, 'fraction_numerical': 0.004376068376068376, 'mean_word_length': 3.4645909645909647, 'pattern_counts': {'":': 0, '<': 143, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5680753', 'n_tokens_mistral': 4415, 'n_tokens_neox': 4255, 'n_words': 1046}