<commit_before>/*
Copyright 2013-present Barefoot Networks, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#include <getopt.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>

#include "setup.h"
#include "options.h"
#include "lib/log.h"
#include "lib/exceptions.h"
#include "lib/nullstream.h"
#include "lib/path.h"
#include "frontends/p4/toP4/toP4.h"
#include "ir/json_generator.h"

static cstring version = "0.0.5";
const char* CompilerOptions::defaultMessage = "Compile a P4 program";

CompilerOptions::CompilerOptions() : Util::Options(defaultMessage) {
    registerOption("-h", nullptr,
                   [this](const char*) { usage(); exit(0); return false; },
                   "Print this help message");
    registerOption("--help", nullptr,
                   [this](const char*) { usage(); exit(0); return false; },
                   "Print this help message");
    registerOption("--version", nullptr,
                   [this](const char*) {
                       std::cerr << binaryName << std::endl;
                       std::cerr << "Version " << version << std::endl;
                       return true; }, "Print compiler version");
    registerOption("-I", "path",
                   [this](const char* arg) {
                       preprocessor_options += std::string(" -I") + arg; return true; },
                   "Specify include path (passed to preprocessor)");
    registerOption("-D", "arg=value",
                   [this](const char* arg) {
                       preprocessor_options += std::string(" -D") + arg; return true; },
                   "Define macro (passed to preprocessor)");
    registerOption("-U", "arg",
                   [this](const char* arg) {
                       preprocessor_options += std::string(" -U") + arg; return true; },
                   "Undefine macro (passed to preprocessor)");
    registerOption("-E", nullptr,
                   [this](const char*) { doNotCompile = true; return true; },
                   "Preprocess only, do not compile (prints program on stdout)");
    registerOption("--p4-14", nullptr,
                   [this](const char*) {
                       langVersion = CompilerOptions::FrontendVersion::P4_14;
                       return true; },
                    "Specify language version to compile");
    registerOption("--p4-16", nullptr,
                   [this](const char*) {
                       langVersion = CompilerOptions::FrontendVersion::P4_16;
                       return true; },
                    "Specify language version to compile");
    registerOption("--p4v", "lang",
                   [this](const char* arg) {
                       if (!strcmp(arg, "1.0") || !strcmp(arg, "14")) {
                           langVersion = CompilerOptions::FrontendVersion::P4_14;
                       } else if (!strcmp(arg, "1.2") || !strcmp(arg, "16")) {
                           langVersion = CompilerOptions::FrontendVersion::P4_16;
                       } else {
                           ::error("Illegal language version %1%", arg);
                           return false;
                       }
                       return true; },
                    "Specify language version to compile");
    registerOption("--target", "target",
                   [this](const char* arg) { target = arg; return true; },
                    "Compile for the specified target");
    registerOption("--pp", "file",
                   [this](const char* arg) { prettyPrintFile = arg; return true; },
                   "Pretty-print the program in the specified file.");
    registerOption("--toJSON", "file",
                   [this](const char* arg) { dumpJsonFile = arg; return true; },
                   "Dump IR to JSON in the specified file.");
    registerOption("--testJson", nullptr,
                    [this](const char*) { debugJson = true; return true; },
                    "Dump and undump the IR");
    registerOption("--p4runtime-file", "file",
                   [this](const char* arg) { p4RuntimeFile = arg; return true; },
                   "Write a P4Runtime control plane API description to the specified file.");
    registerOption("--p4runtime-as-json", nullptr,
                   [this](const char* arg) { p4RuntimeAsJson = true; return true; },
                   "Write out the P4Runtime API description as human-readable JSON.");
    registerOption("-o", "outfile",
                   [this](const char* arg) { outputFile = arg; return true; },
                   "Write output to outfile");
    registerOption("-T", "loglevel",
                   [](const char* arg) { Log::addDebugSpec(arg); return true; },
                   "[Compiler debugging] Adjust logging level per file (see below)");
    registerOption("-v", nullptr,
                   [this](const char*) { Log::increaseVerbosity(); return true; },
                   "[Compiler debugging] Increase verbosity level (can be repeated)");
    registerOption("--top4", "pass1[,pass2]",
                   [this](const char* arg) {
                       auto copy = strdup(arg);
                       while (auto pass = strsep(&copy, ","))
                           top4.push_back(pass);
                       return true;
                   },
                   "[Compiler debugging] Dump the P4 representation after\n"
                   "passes whose name contains one of `passX' substrings.\n"
                   "When '-v' is used this will include the compiler IR.\n");
    registerOption("--dump", "folder",
                   [this](const char* arg) { dumpFolder = arg; return true; },
                   "[Compiler debugging] Folder where P4 programs are dumped\n");
    registerUsage("loglevel format is:\n"
                  "  sourceFile:level,...,sourceFile:level\n"
                  "where 'sourceFile' is a compiler source file and\n"
                  "'level' is the verbosity level for LOG messages in that file");
}

void CompilerOptions::setInputFile() {
    if (remainingOptions.size() > 1) {
        ::error("Only one input file must be specified: %s",
                cstring::join(remainingOptions.begin(), remainingOptions.end(), ","));
        usage();
    } else if (remainingOptions.size() == 0) {
        ::error("No input files specified");
        usage();
    } else {
        file = remainingOptions.at(0);
    }
}

std::vector<const char*>* CompilerOptions::process(int argc, char* const argv[]) {
    char buffer[PATH_MAX];
    int len;
    struct stat st;
    /* find the path of the executable.  We use a number of techniques that may fail
     * or work on different systems, and take the first working one we find.  Fall
     * back to not overriding the compiled-in installation path */
    if ((len = readlink("/proc/self/exe", buffer, sizeof(buffer))) > 0 ||
        (len = readlink("/proc/curproc/exe", buffer, sizeof(buffer))) > 0 ||
        (len = readlink("/proc/curproc/file", buffer, sizeof(buffer))) > 0 ||
        (len = readlink("/proc/self/path/a.out", buffer, sizeof(buffer))) > 0) {
        buffer[len] = 0;
    } else if (argv[0][0] == '/') {
        snprintf(buffer, sizeof(buffer), "%s", argv[0]);
    } else if (strchr(argv[0], '/')) {
        getcwd(buffer, sizeof(buffer));
        strncat(buffer, argv[0], sizeof(buffer) - strlen(buffer) - 1);
    } else if (getenv("_")) {
        strncpy(buffer, getenv("_"), sizeof(buffer));
        buffer[sizeof(buffer) - 1] = 0;
    } else {
        buffer[0] = 0; }

    if (char *p = strrchr(buffer, '/')) {
        ++p;
        exe_name = p;
        snprintf(p, buffer + sizeof(buffer) - p, "p4include");
        if (stat(buffer, &st) >= 0 && S_ISDIR(st.st_mode))
            p4includePath = strdup(buffer);
        snprintf(p, buffer + sizeof(buffer) - p, "p4_14include");
        if (stat(buffer, &st) >= 0 && S_ISDIR(st.st_mode))
            p4_14includePath = strdup(buffer); }

    return Util::Options::process(argc, argv);
}

FILE* CompilerOptions::preprocess() {
    FILE* in = nullptr;

    if (file == "-") {
        file = "<stdin>";
        in = stdin;
    } else {
#ifdef __clang__
        /* FIXME -- while clang has a 'cpp' executable, its broken and doesn't work right, so
         * we need to run clang -E instead.  This should be managed by autoconf (figure out how
         * to portably run the c preprocessor) */
        std::string cmd("cc -E -x c");
#else
        std::string cmd("cpp");
#endif
        cmd += cstring(" -undef -nostdinc") + " " + preprocessor_options
            + " -I" + (isv1() ? p4_14includePath : p4includePath) + " " + file;
        if (Log::verbose())
            std::cerr << "Invoking preprocessor " << std::endl << cmd << std::endl;
        in = popen(cmd.c_str(), "r");
        if (in == nullptr) {
            ::error("Error invoking preprocessor");
            perror("");
            return nullptr;
        }
        close_input = true;
    }

    if (doNotCompile) {
        char *line = NULL;
        size_t len = 0;
        ssize_t read;

        while ((read = getline(&line, &len, in)) != -1)
            printf("%s", line);
        closeInput(in);
        return nullptr;
    }
    return in;
}

void CompilerOptions::closeInput(FILE* inputStream) const {
    if (close_input) {
        int exitCode = pclose(inputStream);
        if (WIFEXITED(exitCode) && WEXITSTATUS(exitCode) == 4)
            ::error("input file %s does not exist", file);
        else if (exitCode != 0)
            ::error("Preprocessor returned exit code %d; aborting compilation", exitCode);
    }
}

// From (folder, file.ext, suffix)  returns
// folder/file-suffix.ext
static cstring makeFileName(cstring folder, cstring name, cstring baseSuffix) {
    Util::PathName filename(name);
    Util::PathName newName(filename.getBasename() + baseSuffix + "." + filename.getExtension());
    auto result = Util::PathName(folder).join(newName.toString());
    return result.toString();
}

bool CompilerOptions::isv1() const {
    return langVersion == CompilerOptions::FrontendVersion::P4_14;
}

void CompilerOptions::dumpPass(const char* manager, unsigned seq, const char* pass,
                               const IR::Node* node) const {
    // Some pass names are currently C++ class names mangled;
    // this is a weak attempt at making them more readable.
    std::string p = pass;
    size_t last = p.find_last_of("0123456789", strlen(pass) - 3);
    if (last != strlen(pass))
        pass = pass + last + 1;
    cstring name = cstring(manager) + "_" + Util::toString(seq) + "_" + pass;
    if (Log::verbose())
        std::cerr << name << std::endl;

    for (auto s : top4) {
        if (strstr(name.c_str(), s.c_str()) != nullptr) {
            cstring suffix = cstring("-") + name;
            cstring filename = file;
            if (filename == "-")
                filename = "tmp.p4";

            cstring fileName = makeFileName(dumpFolder, filename, suffix);
            auto stream = openFile(fileName, true);
            if (stream != nullptr) {
                if (Log::verbose())
                    std::cerr << "Writing program to " << fileName << std::endl;
                P4::ToP4 toP4(stream, Log::verbose(), file);
                node->apply(toP4);
            }
        }
    }
}

DebugHook CompilerOptions::getDebugHook() const {
    auto dp = std::bind(&CompilerOptions::dumpPass, this,
                        std::placeholders::_1, std::placeholders::_2,
                        std::placeholders::_3, std::placeholders::_4);
    return dp;
}
<commit_msg>Provide a directory separator if needed when converting relative to absolute path<commit_after>/*
Copyright 2013-present Barefoot Networks, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#include <getopt.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>

#include "setup.h"
#include "options.h"
#include "lib/log.h"
#include "lib/exceptions.h"
#include "lib/nullstream.h"
#include "lib/path.h"
#include "frontends/p4/toP4/toP4.h"
#include "ir/json_generator.h"

static cstring version = "0.0.5";
const char* CompilerOptions::defaultMessage = "Compile a P4 program";

CompilerOptions::CompilerOptions() : Util::Options(defaultMessage) {
    registerOption("-h", nullptr,
                   [this](const char*) { usage(); exit(0); return false; },
                   "Print this help message");
    registerOption("--help", nullptr,
                   [this](const char*) { usage(); exit(0); return false; },
                   "Print this help message");
    registerOption("--version", nullptr,
                   [this](const char*) {
                       std::cerr << binaryName << std::endl;
                       std::cerr << "Version " << version << std::endl;
                       return true; }, "Print compiler version");
    registerOption("-I", "path",
                   [this](const char* arg) {
                       preprocessor_options += std::string(" -I") + arg; return true; },
                   "Specify include path (passed to preprocessor)");
    registerOption("-D", "arg=value",
                   [this](const char* arg) {
                       preprocessor_options += std::string(" -D") + arg; return true; },
                   "Define macro (passed to preprocessor)");
    registerOption("-U", "arg",
                   [this](const char* arg) {
                       preprocessor_options += std::string(" -U") + arg; return true; },
                   "Undefine macro (passed to preprocessor)");
    registerOption("-E", nullptr,
                   [this](const char*) { doNotCompile = true; return true; },
                   "Preprocess only, do not compile (prints program on stdout)");
    registerOption("--p4-14", nullptr,
                   [this](const char*) {
                       langVersion = CompilerOptions::FrontendVersion::P4_14;
                       return true; },
                    "Specify language version to compile");
    registerOption("--p4-16", nullptr,
                   [this](const char*) {
                       langVersion = CompilerOptions::FrontendVersion::P4_16;
                       return true; },
                    "Specify language version to compile");
    registerOption("--p4v", "lang",
                   [this](const char* arg) {
                       if (!strcmp(arg, "1.0") || !strcmp(arg, "14")) {
                           langVersion = CompilerOptions::FrontendVersion::P4_14;
                       } else if (!strcmp(arg, "1.2") || !strcmp(arg, "16")) {
                           langVersion = CompilerOptions::FrontendVersion::P4_16;
                       } else {
                           ::error("Illegal language version %1%", arg);
                           return false;
                       }
                       return true; },
                    "Specify language version to compile");
    registerOption("--target", "target",
                   [this](const char* arg) { target = arg; return true; },
                    "Compile for the specified target");
    registerOption("--pp", "file",
                   [this](const char* arg) { prettyPrintFile = arg; return true; },
                   "Pretty-print the program in the specified file.");
    registerOption("--toJSON", "file",
                   [this](const char* arg) { dumpJsonFile = arg; return true; },
                   "Dump IR to JSON in the specified file.");
    registerOption("--testJson", nullptr,
                    [this](const char*) { debugJson = true; return true; },
                    "Dump and undump the IR");
    registerOption("--p4runtime-file", "file",
                   [this](const char* arg) { p4RuntimeFile = arg; return true; },
                   "Write a P4Runtime control plane API description to the specified file.");
    registerOption("--p4runtime-as-json", nullptr,
                   [this](const char* arg) { p4RuntimeAsJson = true; return true; },
                   "Write out the P4Runtime API description as human-readable JSON.");
    registerOption("-o", "outfile",
                   [this](const char* arg) { outputFile = arg; return true; },
                   "Write output to outfile");
    registerOption("-T", "loglevel",
                   [](const char* arg) { Log::addDebugSpec(arg); return true; },
                   "[Compiler debugging] Adjust logging level per file (see below)");
    registerOption("-v", nullptr,
                   [this](const char*) { Log::increaseVerbosity(); return true; },
                   "[Compiler debugging] Increase verbosity level (can be repeated)");
    registerOption("--top4", "pass1[,pass2]",
                   [this](const char* arg) {
                       auto copy = strdup(arg);
                       while (auto pass = strsep(&copy, ","))
                           top4.push_back(pass);
                       return true;
                   },
                   "[Compiler debugging] Dump the P4 representation after\n"
                   "passes whose name contains one of `passX' substrings.\n"
                   "When '-v' is used this will include the compiler IR.\n");
    registerOption("--dump", "folder",
                   [this](const char* arg) { dumpFolder = arg; return true; },
                   "[Compiler debugging] Folder where P4 programs are dumped\n");
    registerUsage("loglevel format is:\n"
                  "  sourceFile:level,...,sourceFile:level\n"
                  "where 'sourceFile' is a compiler source file and\n"
                  "'level' is the verbosity level for LOG messages in that file");
}

void CompilerOptions::setInputFile() {
    if (remainingOptions.size() > 1) {
        ::error("Only one input file must be specified: %s",
                cstring::join(remainingOptions.begin(), remainingOptions.end(), ","));
        usage();
    } else if (remainingOptions.size() == 0) {
        ::error("No input files specified");
        usage();
    } else {
        file = remainingOptions.at(0);
    }
}

template <size_t N>
static void convertToAbsPath(const char* const relPath, char (&output)[N]) {
    output[0] = '\0';  // Default to the empty string, indicating failure.

    char cwd[PATH_MAX];
    getcwd(cwd, sizeof(cwd));
    const size_t cwdLen = strlen(cwd);
    if (cwdLen == 0) return;
    const char* separator = cwd[cwdLen - 1] == '/' ? "" : "/";

    // Construct an absolute path. We're assuming that @relPath is relative to
    // the current working directory.
    snprintf(output, N, "%s%s%s", cwd, separator, relPath);
}

std::vector<const char*>* CompilerOptions::process(int argc, char* const argv[]) {
    char buffer[PATH_MAX];
    int len;
    struct stat st;
    /* find the path of the executable.  We use a number of techniques that may fail
     * or work on different systems, and take the first working one we find.  Fall
     * back to not overriding the compiled-in installation path */
    if ((len = readlink("/proc/self/exe", buffer, sizeof(buffer))) > 0 ||
        (len = readlink("/proc/curproc/exe", buffer, sizeof(buffer))) > 0 ||
        (len = readlink("/proc/curproc/file", buffer, sizeof(buffer))) > 0 ||
        (len = readlink("/proc/self/path/a.out", buffer, sizeof(buffer))) > 0) {
        buffer[len] = 0;
    } else if (argv[0][0] == '/') {
        snprintf(buffer, sizeof(buffer), "%s", argv[0]);
    } else if (strchr(argv[0], '/')) {
        convertToAbsPath(argv[0], buffer);
    } else if (getenv("_")) {
        strncpy(buffer, getenv("_"), sizeof(buffer));
        buffer[sizeof(buffer) - 1] = 0;
    } else {
        buffer[0] = 0; }

    if (char *p = strrchr(buffer, '/')) {
        ++p;
        exe_name = p;
        snprintf(p, buffer + sizeof(buffer) - p, "p4include");
        if (stat(buffer, &st) >= 0 && S_ISDIR(st.st_mode))
            p4includePath = strdup(buffer);
        snprintf(p, buffer + sizeof(buffer) - p, "p4_14include");
        if (stat(buffer, &st) >= 0 && S_ISDIR(st.st_mode))
            p4_14includePath = strdup(buffer); }

    return Util::Options::process(argc, argv);
}

FILE* CompilerOptions::preprocess() {
    FILE* in = nullptr;

    if (file == "-") {
        file = "<stdin>";
        in = stdin;
    } else {
#ifdef __clang__
        /* FIXME -- while clang has a 'cpp' executable, its broken and doesn't work right, so
         * we need to run clang -E instead.  This should be managed by autoconf (figure out how
         * to portably run the c preprocessor) */
        std::string cmd("cc -E -x c");
#else
        std::string cmd("cpp");
#endif
        cmd += cstring(" -undef -nostdinc") + " " + preprocessor_options
            + " -I" + (isv1() ? p4_14includePath : p4includePath) + " " + file;
        if (Log::verbose())
            std::cerr << "Invoking preprocessor " << std::endl << cmd << std::endl;
        in = popen(cmd.c_str(), "r");
        if (in == nullptr) {
            ::error("Error invoking preprocessor");
            perror("");
            return nullptr;
        }
        close_input = true;
    }

    if (doNotCompile) {
        char *line = NULL;
        size_t len = 0;
        ssize_t read;

        while ((read = getline(&line, &len, in)) != -1)
            printf("%s", line);
        closeInput(in);
        return nullptr;
    }
    return in;
}

void CompilerOptions::closeInput(FILE* inputStream) const {
    if (close_input) {
        int exitCode = pclose(inputStream);
        if (WIFEXITED(exitCode) && WEXITSTATUS(exitCode) == 4)
            ::error("input file %s does not exist", file);
        else if (exitCode != 0)
            ::error("Preprocessor returned exit code %d; aborting compilation", exitCode);
    }
}

// From (folder, file.ext, suffix)  returns
// folder/file-suffix.ext
static cstring makeFileName(cstring folder, cstring name, cstring baseSuffix) {
    Util::PathName filename(name);
    Util::PathName newName(filename.getBasename() + baseSuffix + "." + filename.getExtension());
    auto result = Util::PathName(folder).join(newName.toString());
    return result.toString();
}

bool CompilerOptions::isv1() const {
    return langVersion == CompilerOptions::FrontendVersion::P4_14;
}

void CompilerOptions::dumpPass(const char* manager, unsigned seq, const char* pass,
                               const IR::Node* node) const {
    // Some pass names are currently C++ class names mangled;
    // this is a weak attempt at making them more readable.
    std::string p = pass;
    size_t last = p.find_last_of("0123456789", strlen(pass) - 3);
    if (last != strlen(pass))
        pass = pass + last + 1;
    cstring name = cstring(manager) + "_" + Util::toString(seq) + "_" + pass;
    if (Log::verbose())
        std::cerr << name << std::endl;

    for (auto s : top4) {
        if (strstr(name.c_str(), s.c_str()) != nullptr) {
            cstring suffix = cstring("-") + name;
            cstring filename = file;
            if (filename == "-")
                filename = "tmp.p4";

            cstring fileName = makeFileName(dumpFolder, filename, suffix);
            auto stream = openFile(fileName, true);
            if (stream != nullptr) {
                if (Log::verbose())
                    std::cerr << "Writing program to " << fileName << std::endl;
                P4::ToP4 toP4(stream, Log::verbose(), file);
                node->apply(toP4);
            }
        }
    }
}

DebugHook CompilerOptions::getDebugHook() const {
    auto dp = std::bind(&CompilerOptions::dumpPass, this,
                        std::placeholders::_1, std::placeholders::_2,
                        std::placeholders::_3, std::placeholders::_4);
    return dp;
}

{'fraction_non_alphanumeric': 0.1384209458088086, 'fraction_numerical': 0.009864014613354982, 'mean_word_length': 2.110606060606061, 'pattern_counts': {'":': 0, '<': 72, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2733767', 'n_tokens_mistral': 7037, 'n_tokens_neox': 6764, 'n_words': 1998}