<commit_before># BenchExec is a framework for reliable benchmarking.
# This file is part of BenchExec.
#
# Copyright (C) 2007-2016  <NAME>
# All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Utility functions for implementing a container using Linux namespaces
and for appropriately configuring such a container."""

# prepare for Python 3
from __future__ import absolute_import, division, print_function, unicode_literals

# THIS MODULE HAS TO WORK WITH PYTHON 2.7!

import contextlib
import ctypes
import errno
import fcntl
import logging
import os
import signal
import socket
import struct

from benchexec import libc
from benchexec import util

__all__ = [
    "execute_in_namespace",
    "setup_user_mapping",
    "activate_network_interface",
    "duplicate_mount_hierarchy",
    "determine_directory_mode",
    "get_mount_points",
    "remount_with_additional_flags",
    "make_overlay_mount",
    "mount_proc",
    "make_bind_mount",
    "get_my_pid_from_procfs",
    "drop_capabilities",
    "forward_all_signals_async",
    "wait_for_child_and_forward_signals",
    "setup_container_system_config",
    "CONTAINER_UID",
    "CONTAINER_GID",
    "CONTAINER_HOME",
    "CONTAINER_HOSTNAME",
]


DEFAULT_STACK_SIZE = 1024 * 1024
GUARD_PAGE_SIZE = 4096  # size of guard page at end of stack

CONTAINER_UID = 1000
CONTAINER_GID = 1000
CONTAINER_HOME = "/home/benchexec"
CONTAINER_HOSTNAME = "benchexec"

CONTAINER_ETC_NSSWITCH_CONF = """
passwd: files
group: files
shadow: files
hosts: files
networks: files

protocols:      db files
services:       db files
ethers:         db files
rpc:            db files

netgroup:       files
automount:      files
"""
CONTAINER_ETC_PASSWD = """
root:x:0:0:root:/root:/bin/bash
benchexec:x:{uid}:{gid}:benchexec:{home}:/bin/bash
nobody:x:65534:65534:nobody:/:/bin/false
""".format(
    uid=CONTAINER_UID, gid=CONTAINER_GID, home=CONTAINER_HOME
)

CONTAINER_ETC_GROUP = """
root:x:0:
benchexec:x:{gid}:
nogroup:x:65534:
""".format(
    uid=CONTAINER_UID, gid=CONTAINER_GID, home=CONTAINER_HOME
)

CONTAINER_ETC_HOSTS = """
127.0.0.1       localhost {host}
# The following lines are desirable for IPv6 capable hosts
::1     localhost ip6-localhost ip6-loopback
fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b ip6-allnodes
fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b ip6-allrouters
""".format(
    host=CONTAINER_HOSTNAME
)

CONTAINER_ETC_FILE_OVERRIDE = {
    b"nsswitch.conf": CONTAINER_ETC_NSSWITCH_CONF,
    b"passwd": CONTAINER_ETC_PASSWD,
    b"group": CONTAINER_ETC_GROUP,
    b"hostname": CONTAINER_HOSTNAME + "\n",
    b"hosts": CONTAINER_ETC_HOSTS,
}

DIR_HIDDEN = "hidden"
DIR_READ_ONLY = "read-only"
DIR_OVERLAY = "overlay"
DIR_FULL_ACCESS = "full-access"
DIR_MODES = [DIR_HIDDEN, DIR_READ_ONLY, DIR_OVERLAY, DIR_FULL_ACCESS]
"""modes how a directory can be mounted in the container"""

LXCFS_PROC_DIR = b"/var/lib/lxcfs/proc"


@contextlib.contextmanager
def allocate_stack(size=DEFAULT_STACK_SIZE):
    """Allocate some memory that can be used as a stack.
    @return: a ctypes void pointer to the *top* of the stack.
    """
    # Allocate memory with appropriate flags for a stack as in
    # https://blog.fefe.de/?ts=a85c8ba7
    base = libc.mmap(
        None,
        size + GUARD_PAGE_SIZE,
        libc.PROT_READ | libc.PROT_WRITE,
        libc.MAP_PRIVATE | libc.MAP_ANONYMOUS | libc.MAP_GROWSDOWN | libc.MAP_STACK,
        -1,
        0,
    )

    try:
        # create a guard page that crashes the application when it is written to
        # (on stack overflow)
        libc.mprotect(base, GUARD_PAGE_SIZE, libc.PROT_NONE)

        yield ctypes.c_void_p(base + size + GUARD_PAGE_SIZE)
    finally:
        libc.munmap(base, size + GUARD_PAGE_SIZE)


def execute_in_namespace(func, use_network_ns=True):
    """Execute a function in a child process in separate namespaces.
    @param func: a parameter-less function returning an int
        (which will be the process' exit value)
    @return: the PID of the created child process
    """
    flags = (
        signal.SIGCHLD
        | libc.CLONE_NEWNS
        | libc.CLONE_NEWUTS
        | libc.CLONE_NEWIPC
        | libc.CLONE_NEWUSER
        | libc.CLONE_NEWPID
    )
    if use_network_ns:
        flags |= libc.CLONE_NEWNET

    # We use the syscall clone() here, which is similar to fork().
    # Calling it without letting Python know about it is dangerous (especially because
    # we want to execute Python code in the child, too), but so far it seems to work.
    # Basically we attempt to do (almost) the same that os.fork() does (cf. os_fork_impl
    # in https://github.com/python/cpython/blob/master/Modules/posixmodule.c).
    # We currently do not take the import lock os.lock() does because it is only
    # available via an internal API, and because the child should never import anything
    # anyway (inside the container, modules might not be visible).
    # It is very important, however, that we have the GIL during clone(),
    # otherwise the child will often deadlock when trying to execute Python code.
    # Luckily, the ctypes module allows us to hold the GIL while executing the
    # function by using ctypes.PyDLL as library access instead of ctypes.CLL.

    def child_func():
        # This is necessary for correcting the Python interpreter state after a
        # fork-like operation. For example, it resets the GIL and fixes state of
        # several modules like threading and signal.
        ctypes.pythonapi.PyOS_AfterFork()

        return func()

    with allocate_stack() as stack:
        pid = libc.clone(ctypes.CFUNCTYPE(ctypes.c_int)(child_func), stack, flags, None)
    return pid


def setup_user_mapping(
    pid,
    uid=os.getuid(),
    gid=os.getgid(),
    parent_uid=os.getuid(),
    parent_gid=os.getgid(),
):
    """Write uid_map and gid_map in /proc to create a user mapping
    that maps our user from outside the container to the same user inside the container
    (and no other users are mapped).
    @see: http://man7.org/linux/man-pages/man7/user_namespaces.7.html
    @param pid: The PID of the process in the container.
    @param uid: The UID that shall be used in the container.
    @param gid: The GID that shall be used in the container.
    @param parent_uid: The UID that is used in the parent namespace.
    @param parent_gid: The GID that is used in the parent namespace.
    """
    proc_child = os.path.join("/proc", str(pid))
    try:
        # map uid internally to our uid externally
        uid_map = "{0} {1} 1".format(uid, parent_uid)
        util.write_file(uid_map, proc_child, "uid_map")
    except IOError as e:
        logging.warning("Creating UID mapping into container failed: %s", e)<commit_msg>Fix container mode on Python >= 3.7<commit_after># BenchExec is a framework for reliable benchmarking.
# This file is part of BenchExec.
#
# Copyright (C) 2007-2016  <NAME>
# All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Utility functions for implementing a container using Linux namespaces
and for appropriately configuring such a container."""

# prepare for Python 3
from __future__ import absolute_import, division, print_function, unicode_literals

# THIS MODULE HAS TO WORK WITH PYTHON 2.7!

import contextlib
import ctypes
import errno
import fcntl
import logging
import os
import signal
import socket
import struct

from benchexec import libc
from benchexec import util

__all__ = [
    "execute_in_namespace",
    "setup_user_mapping",
    "activate_network_interface",
    "duplicate_mount_hierarchy",
    "determine_directory_mode",
    "get_mount_points",
    "remount_with_additional_flags",
    "make_overlay_mount",
    "mount_proc",
    "make_bind_mount",
    "get_my_pid_from_procfs",
    "drop_capabilities",
    "forward_all_signals_async",
    "wait_for_child_and_forward_signals",
    "setup_container_system_config",
    "CONTAINER_UID",
    "CONTAINER_GID",
    "CONTAINER_HOME",
    "CONTAINER_HOSTNAME",
]


DEFAULT_STACK_SIZE = 1024 * 1024
GUARD_PAGE_SIZE = 4096  # size of guard page at end of stack

CONTAINER_UID = 1000
CONTAINER_GID = 1000
CONTAINER_HOME = "/home/benchexec"
CONTAINER_HOSTNAME = "benchexec"

CONTAINER_ETC_NSSWITCH_CONF = """
passwd: files
group: files
shadow: files
hosts: files
networks: files

protocols:      db files
services:       db files
ethers:         db files
rpc:            db files

netgroup:       files
automount:      files
"""
CONTAINER_ETC_PASSWD = """
root:x:0:0:root:/root:/bin/bash
benchexec:x:{uid}:{gid}:benchexec:{home}:/bin/bash
nobody:x:65534:65534:nobody:/:/bin/false
""".format(
    uid=CONTAINER_UID, gid=CONTAINER_GID, home=CONTAINER_HOME
)

CONTAINER_ETC_GROUP = """
root:x:0:
benchexec:x:{gid}:
nogroup:x:65534:
""".format(
    uid=CONTAINER_UID, gid=CONTAINER_GID, home=CONTAINER_HOME
)

CONTAINER_ETC_HOSTS = """
127.0.0.1       localhost {host}
# The following lines are desirable for IPv6 capable hosts
::1     localhost ip6-localhost ip6-loopback
fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b ip6-allnodes
fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b ip6-allrouters
""".format(
    host=CONTAINER_HOSTNAME
)

CONTAINER_ETC_FILE_OVERRIDE = {
    b"nsswitch.conf": CONTAINER_ETC_NSSWITCH_CONF,
    b"passwd": <PASSWORD>,
    b"group": CONTAINER_ETC_GROUP,
    b"hostname": CONTAINER_HOSTNAME + "\n",
    b"hosts": CONTAINER_ETC_HOSTS,
}

DIR_HIDDEN = "hidden"
DIR_READ_ONLY = "read-only"
DIR_OVERLAY = "overlay"
DIR_FULL_ACCESS = "full-access"
DIR_MODES = [DIR_HIDDEN, DIR_READ_ONLY, DIR_OVERLAY, DIR_FULL_ACCESS]
"""modes how a directory can be mounted in the container"""

LXCFS_PROC_DIR = b"/var/lib/lxcfs/proc"

# Python before 3.7 does not have BeforeFork and AfterFork_(Child|Parent)
if not hasattr(ctypes.pythonapi, "PyOS_BeforeFork"):
    ctypes.pythonapi.PyOS_BeforeFork = lambda: None
if not hasattr(ctypes.pythonapi, "PyOS_AfterFork_Parent"):
    ctypes.pythonapi.PyOS_AfterFork_Parent = lambda: None
if not hasattr(ctypes.pythonapi, "PyOS_AfterFork_Child"):
    ctypes.pythonapi.PyOS_AfterFork_Child = ctypes.pythonapi.PyOS_AfterFork


@contextlib.contextmanager
def allocate_stack(size=DEFAULT_STACK_SIZE):
    """Allocate some memory that can be used as a stack.
    @return: a ctypes void pointer to the *top* of the stack.
    """
    # Allocate memory with appropriate flags for a stack as in
    # https://blog.fefe.de/?ts=a85c8ba7
    base = libc.mmap(
        None,
        size + GUARD_PAGE_SIZE,
        libc.PROT_READ | libc.PROT_WRITE,
        libc.MAP_PRIVATE | libc.MAP_ANONYMOUS | libc.MAP_GROWSDOWN | libc.MAP_STACK,
        -1,
        0,
    )

    try:
        # create a guard page that crashes the application when it is written to
        # (on stack overflow)
        libc.mprotect(base, GUARD_PAGE_SIZE, libc.PROT_NONE)

        yield ctypes.c_void_p(base + size + GUARD_PAGE_SIZE)
    finally:
        libc.munmap(base, size + GUARD_PAGE_SIZE)


def execute_in_namespace(func, use_network_ns=True):
    """Execute a function in a child process in separate namespaces.
    @param func: a parameter-less function returning an int
        (which will be the process' exit value)
    @return: the PID of the created child process
    """
    flags = (
        signal.SIGCHLD
        | libc.CLONE_NEWNS
        | libc.CLONE_NEWUTS
        | libc.CLONE_NEWIPC
        | libc.CLONE_NEWUSER
        | libc.CLONE_NEWPID
    )
    if use_network_ns:
        flags |= libc.CLONE_NEWNET

    # We use the syscall clone() here, which is similar to fork().
    # Calling it without letting Python know about it is dangerous (especially because
    # we want to execute Python code in the child, too), but so far it seems to work.
    # Basically we attempt to do (almost) the same that os.fork() does (cf. os_fork_impl
    # in https://github.com/python/cpython/blob/master/Modules/posixmodule.c).
    # On Python >= 3.7 we can call appropriate functions before and after fork that
    # should handle everything correctly, on previous Python we do the best that we can.
    # For example, we do not take the import lock because it is only
    # available via an internal API, and because the child should never import anything
    # anyway (inside the container, modules might not be visible).
    # It is very important, however, that we have the GIL during clone(),
    # otherwise the child will often deadlock when trying to execute Python code.
    # Luckily, the ctypes module allows us to hold the GIL while executing the
    # function by using ctypes.PyDLL as library access instead of ctypes.CLL.

    def child_func():
        ctypes.pythonapi.PyOS_AfterFork_Child()

        return func()

    with allocate_stack() as stack:
        try:
            ctypes.pythonapi.PyOS_BeforeFork()
            pid = libc.clone(
                ctypes.CFUNCTYPE(ctypes.c_int)(child_func), stack, flags, None
            )
        finally:
            ctypes.pythonapi.PyOS_AfterFork_Parent()
    return pid


def setup_user_mapping(
    pid,
    uid=os.getuid(),
    gid=os.getgid(),
    parent_uid=os.getuid(),
    parent_gid=os.getgid(),
):
    """Write uid_map and gid_map in /proc to create a user mapping
    that maps our user from outside the container to the same user inside the container
    (and no other users are mapped).
    @see: http://man7.org/linux/man-pages/man7/user_namespaces.7.html
    @param pid: The PID of the process in the container.
    @param uid: The UID that shall be used in the container.
    @param gid: The GID that shall be used in the container.
    @param parent_uid: The UID that is used in the parent namespace.
    @param parent_gid: The GID that is used in the parent namespace.
    """
    proc_child = os.path.join("/proc", str(pid))
    try:
        # map uid internally to our uid externally
        uid_map = "{0} {1} 1".format(uid, parent_uid)
        util.write_file(uid_map, proc_child, "uid_map")
    except IOError as e:
        logging.warning("Creating UID mapping into container failed: %s", e)
{'fraction_non_alphanumeric': 0.0881057268722467, 'fraction_numerical': 0.015752236016553196, 'mean_word_length': 3.541679296756593, 'pattern_counts': {'":': 10, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1964551', 'n_tokens_mistral': 5176, 'n_tokens_neox': 4854, 'n_words': 1507}