<commit_before>/* runner.cc                                                       -*- C++ -*-
   <NAME>, September 2013
   Copyright (c) 2013 Datacratic.  All rights reserved.

   A command runner class that hides the specifics of the underlying unix
   system calls and can intercept input and output.
*/

#include <fcntl.h>
#include <poll.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/epoll.h>
#include <sys/prctl.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/resource.h>

#include <iostream>
#include <utility>

#include "jml/arch/futex.h"
#include "jml/arch/timers.h"
#include "jml/utils/file_functions.h"

#include "message_loop.h"
#include "sink.h"

#include "runner.h"

using namespace std;
using namespace Datacratic;


namespace {

struct ChildStatus
{
    ChildStatus()
        : pid(-1)
    {}
    union {
        pid_t pid;
        int status;
    };
};

tuple<int, int>
CreateStdPipe(bool forWriting)
{
    int fds[2];
    int rc = pipe(fds);
    if (rc == -1) {
        throw ML::Exception(errno, "CreateStdPipe pipe2");
    }

    if (forWriting) {
        return tuple<int, int>(fds[1], fds[0]);
    }
    else {
        return tuple<int, int>(fds[0], fds[1]);
    }
}

struct ChildFds {
    ChildFds()
        : stdIn(::fileno(stdin)),
          stdOut(::fileno(stdout)),
          stdErr(::fileno(stderr)),
          statusFd(-1)
    {
    }

    /* child api */
    void closeRemainingFds()
    {
        struct rlimit limits;
        ::getrlimit(RLIMIT_NOFILE, &limits);

        for (int fd = 0; fd < limits.rlim_cur; fd++) {
            if (fd != STDIN_FILENO
                && fd != STDOUT_FILENO && fd != STDERR_FILENO
                && fd != statusFd) {
                ::close(fd);
            }
        }
    }

    void dupToStdStreams()
    {
        auto dupToStdStream = [&] (int oldFd, int newFd) {
            if (oldFd != newFd) {
                int rc = ::dup2(oldFd, newFd);
                if (rc == -1) {
                    throw ML::Exception(errno,
                                        "ChildFds::dupToStdStream dup2");
                }
            }
        };
        dupToStdStream(stdIn, STDIN_FILENO);
        dupToStdStream(stdOut, STDOUT_FILENO);
        dupToStdStream(stdErr, STDERR_FILENO);
    }

    /* parent & child api */
    void close()
    {
        auto closeIfNotEqual = [&] (int & fd, int notValue) {
            if (fd != notValue) {
                ::close(fd);
            }
        };
        closeIfNotEqual(stdIn, STDIN_FILENO);
        closeIfNotEqual(stdOut, STDOUT_FILENO);
        closeIfNotEqual(stdErr, STDERR_FILENO);
        closeIfNotEqual(statusFd, -1);
    }

    int stdIn;
    int stdOut;
    int stdErr;
    int statusFd;
};

void
RunWrapper(const vector<string> & command, ChildFds & fds)
{
    // Undo any SIGCHLD block from the parent process so it can
    // properly wait for the signal
    ::signal(SIGCHLD, SIG_DFL);
    ::signal(SIGPIPE, SIG_DFL);

    fds.dupToStdStreams();
    fds.closeRemainingFds();

    // Set up the arguments before we fork, as we don't want to call malloc()
    // from the fork, and it can be called from c_str() in theory.
    size_t len = command.size();
    char * argv[len + 1];
    for (int i = 0; i < len; i++) {
        argv[i] = (char *) command[i].c_str();
    }
    argv[len] = NULL;

    int childPid = fork();
    if (childPid == -1) {
        throw ML::Exception(errno, "RunWrapper fork");
    }
    else if (childPid == 0) {
        ::prctl(PR_SET_PDEATHSIG, SIGTERM);
        ::close(fds.statusFd);
        int res = ::execv(command[0].c_str(), argv);
        if (res == -1) {
            throw ML::Exception(errno, "RunWrapper exec");
        }

        /* there is no possible way this code could be executed */
        throw ML::Exception("The Alpha became the Omega.");
    }
    else {
        // FILE * terminal = ::fopen("/dev/tty", "a");

        // ::fprintf(terminal, "wrapper: real child pid: %d\n", childPid);
        ChildStatus status;

        status.pid = childPid;
        if (::write(fds.statusFd, &status, sizeof(status)) == -1) {
            throw ML::Exception(errno, "RunWrapper write");
        }

        // ::fprintf(terminal, "wrapper: waiting child...\n");

        int res = ::waitpid(childPid, &status.status, 0);
        if (res == -1)
            throw ML::Exception(errno, "waitpid");
        if (res != childPid)
            throw ML::Exception("waitpid has not returned the childPid");

        // ::fprintf(terminal, "wrapper: child terminated\n");

        if (::write(fds.statusFd, &status, sizeof(status)) == -1) {
            throw ML::Exception(errno, "RunWrapper write");
        }

        fds.close();

        ::exit(0);
    }
}

} // namespace

<commit_msg>Runner: exit from the wrapper process using _exit, to void executing functions registered from the parent via atexit<commit_after>/* runner.cc                                                       -*- C++ -*-
   <NAME>, September 2013
   Copyright (c) 2013 Datacratic.  All rights reserved.

   A command runner class that hides the specifics of the underlying unix
   system calls and can intercept input and output.
*/

#include <fcntl.h>
#include <poll.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/epoll.h>
#include <sys/prctl.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/resource.h>

#include <iostream>
#include <utility>

#include "jml/arch/futex.h"
#include "jml/arch/timers.h"
#include "jml/utils/file_functions.h"

#include "message_loop.h"
#include "sink.h"

#include "runner.h"

using namespace std;
using namespace Datacratic;


namespace {

struct ChildStatus
{
    ChildStatus()
        : pid(-1)
    {}
    union {
        pid_t pid;
        int status;
    };
};

tuple<int, int>
CreateStdPipe(bool forWriting)
{
    int fds[2];
    int rc = pipe(fds);
    if (rc == -1) {
        throw ML::Exception(errno, "CreateStdPipe pipe2");
    }

    if (forWriting) {
        return tuple<int, int>(fds[1], fds[0]);
    }
    else {
        return tuple<int, int>(fds[0], fds[1]);
    }
}

struct ChildFds {
    ChildFds()
        : stdIn(::fileno(stdin)),
          stdOut(::fileno(stdout)),
          stdErr(::fileno(stderr)),
          statusFd(-1)
    {
    }

    /* child api */
    void closeRemainingFds()
    {
        struct rlimit limits;
        ::getrlimit(RLIMIT_NOFILE, &limits);

        for (int fd = 0; fd < limits.rlim_cur; fd++) {
            if (fd != STDIN_FILENO
                && fd != STDOUT_FILENO && fd != STDERR_FILENO
                && fd != statusFd) {
                ::close(fd);
            }
        }
    }

    void dupToStdStreams()
    {
        auto dupToStdStream = [&] (int oldFd, int newFd) {
            if (oldFd != newFd) {
                int rc = ::dup2(oldFd, newFd);
                if (rc == -1) {
                    throw ML::Exception(errno,
                                        "ChildFds::dupToStdStream dup2");
                }
            }
        };
        dupToStdStream(stdIn, STDIN_FILENO);
        dupToStdStream(stdOut, STDOUT_FILENO);
        dupToStdStream(stdErr, STDERR_FILENO);
    }

    /* parent & child api */
    void close()
    {
        auto closeIfNotEqual = [&] (int & fd, int notValue) {
            if (fd != notValue) {
                ::close(fd);
            }
        };
        closeIfNotEqual(stdIn, STDIN_FILENO);
        closeIfNotEqual(stdOut, STDOUT_FILENO);
        closeIfNotEqual(stdErr, STDERR_FILENO);
        closeIfNotEqual(statusFd, -1);
    }

    int stdIn;
    int stdOut;
    int stdErr;
    int statusFd;
};

void
RunWrapper(const vector<string> & command, ChildFds & fds)
{
    // Undo any SIGCHLD block from the parent process so it can
    // properly wait for the signal
    ::signal(SIGCHLD, SIG_DFL);
    ::signal(SIGPIPE, SIG_DFL);

    fds.dupToStdStreams();
    fds.closeRemainingFds();

    // Set up the arguments before we fork, as we don't want to call malloc()
    // from the fork, and it can be called from c_str() in theory.
    size_t len = command.size();
    char * argv[len + 1];
    for (int i = 0; i < len; i++) {
        argv[i] = (char *) command[i].c_str();
    }
    argv[len] = NULL;

    int childPid = fork();
    if (childPid == -1) {
        throw ML::Exception(errno, "RunWrapper fork");
    }
    else if (childPid == 0) {
        ::prctl(PR_SET_PDEATHSIG, SIGTERM);
        ::close(fds.statusFd);
        int res = ::execv(command[0].c_str(), argv);
        if (res == -1) {
            throw ML::Exception(errno, "RunWrapper exec");
        }

        /* there is no possible way this code could be executed */
        throw ML::Exception("The Alpha became the Omega.");
    }
    else {
        // FILE * terminal = ::fopen("/dev/tty", "a");

        // ::fprintf(terminal, "wrapper: real child pid: %d\n", childPid);
        ChildStatus status;

        status.pid = childPid;
        if (::write(fds.statusFd, &status, sizeof(status)) == -1) {
            throw ML::Exception(errno, "RunWrapper write");
        }

        // ::fprintf(terminal, "wrapper: waiting child...\n");

        int res = ::waitpid(childPid, &status.status, 0);
        if (res == -1)
            throw ML::Exception(errno, "waitpid");
        if (res != childPid)
            throw ML::Exception("waitpid has not returned the childPid");

        // ::fprintf(terminal, "wrapper: child terminated\n");

        if (::write(fds.statusFd, &status, sizeof(status)) == -1) {
            throw ML::Exception(errno, "RunWrapper write");
        }

        fds.close();

        ::_exit(0);
    }
}

} // namespace


{'fraction_non_alphanumeric': 0.14300348289284984, 'fraction_numerical': 0.006760909649661954, 'mean_word_length': 2.020730198019802, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2594416', 'n_tokens_mistral': 3325, 'n_tokens_neox': 3219, 'n_words': 797}