<commit_before>"""
Miscellaneous utility functions, wrappers around some subprocess procedures.
"""

import concurrent.futures
import fcntl
import hashlib
import importlib
import inspect
import json
import logging
import logging.config
import os
import random
import shlex
import sys
from subprocess import PIPE, STDOUT, Popen
from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
    Dict,
    Generic,
    Iterable,
    List,
    Optional,
    Tuple,
    Type,
    TypeVar,
    Union,
)

import netaddr

from core.errors import CoreCommandError, CoreError

if TYPE_CHECKING:
    from core.emulator.session import Session
    from core.nodes.base import CoreNode
T = TypeVar("T")

DEVNULL = open(os.devnull, "wb")


def execute_file(
    path: str, exec_globals: Dict[str, str] = None, exec_locals: Dict[str, str] = None
) -> None:
    """
    Provides an alternative way to run execfile to be compatible for
    both python2/3.

    :param path: path of file to execute
    :param exec_globals: globals values to pass to execution
    :param exec_locals:  local values to pass to execution
    :return: nothing
    """
    if exec_globals is None:
        exec_globals = {}
    exec_globals.update({"__file__": path, "__name__": "__main__"})
    with open(path, "rb") as f:
        data = compile(f.read(), path, "exec")
        exec(data, exec_globals, exec_locals)


def hashkey(value: Union[str, int]) -> int:
    """
    Provide a consistent hash that can be used in place
    of the builtin hash, that no longer behaves consistently
    in python3.

    :param value: value to hash
    :return: hash value
    """
    if isinstance(value, int):
        value = str(value)
    value = value.encode("utf-8")
    return int(hashlib.sha256(value).hexdigest(), 16)


def _detach_init() -> None:
    """
    Fork a child process and exit.

    :return: nothing
    """
    if os.fork():
        # parent exits
        os._exit(0)
    os.setsid()


def _valid_module(path: str, file_name: str) -> bool:
    """
    Check if file is a valid python module.

    :param path: path to file
    :param file_name: file name to check
    :return: True if a valid python module file, False otherwise
    """
    file_path = os.path.join(path, file_name)
    if not os.path.isfile(file_path):
        return False

    if file_name.startswith("_"):
        return False

    if not file_name.endswith(".py"):
        return False

    return True


def _is_class(module: Any, member: Type, clazz: Type) -> bool:
    """
    Validates if a module member is a class and an instance of a CoreService.

    :param module: module to validate for service
    :param member: member to validate for service
    :param clazz: clazz type to check for validation
    :return: True if a valid service, False otherwise
    """
    if not inspect.isclass(member):
        return False

    if not issubclass(member, clazz):
        return False

    if member.__module__ != module.__name__:
        return False

    return True


def close_onexec(fd: int) -> None:
    """
    Close on execution of a shell process.

    :param fd: file descriptor to close
    :return: nothing
    """
    fdflags = fcntl.fcntl(fd, fcntl.F_GETFD)
    fcntl.fcntl(fd, fcntl.F_SETFD, fdflags | fcntl.FD_CLOEXEC)


def which(command: str, required: bool) -> str:
    """
    Find location of desired executable within current PATH.

    :param command: command to find location for
    :param required: command is required to be found, false otherwise
    :return: command location or None
    :raises ValueError: when not found and required
    """
    found_path = None
    for path in os.environ["PATH"].split(os.pathsep):
        command_path = os.path.join(path, command)
        if os.path.isfile(command_path) and os.access(command_path, os.X_OK):
            found_path = command_path
            break

    if found_path is None and required:
        raise ValueError(f"failed to find required executable({command}) in path")

    return found_path<commit_msg>update to make use of shutil.which for executable searching<commit_after>"""
Miscellaneous utility functions, wrappers around some subprocess procedures.
"""

import concurrent.futures
import fcntl
import hashlib
import importlib
import inspect
import json
import logging
import logging.config
import os
import random
import shlex
import shutil
import sys
from subprocess import PIPE, STDOUT, Popen
from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
    Dict,
    Generic,
    Iterable,
    List,
    Optional,
    Tuple,
    Type,
    TypeVar,
    Union,
)

import netaddr

from core.errors import CoreCommandError, CoreError

if TYPE_CHECKING:
    from core.emulator.session import Session
    from core.nodes.base import CoreNode
T = TypeVar("T")

DEVNULL = open(os.devnull, "wb")


def execute_file(
    path: str, exec_globals: Dict[str, str] = None, exec_locals: Dict[str, str] = None
) -> None:
    """
    Provides an alternative way to run execfile to be compatible for
    both python2/3.

    :param path: path of file to execute
    :param exec_globals: globals values to pass to execution
    :param exec_locals:  local values to pass to execution
    :return: nothing
    """
    if exec_globals is None:
        exec_globals = {}
    exec_globals.update({"__file__": path, "__name__": "__main__"})
    with open(path, "rb") as f:
        data = compile(f.read(), path, "exec")
        exec(data, exec_globals, exec_locals)


def hashkey(value: Union[str, int]) -> int:
    """
    Provide a consistent hash that can be used in place
    of the builtin hash, that no longer behaves consistently
    in python3.

    :param value: value to hash
    :return: hash value
    """
    if isinstance(value, int):
        value = str(value)
    value = value.encode("utf-8")
    return int(hashlib.sha256(value).hexdigest(), 16)


def _detach_init() -> None:
    """
    Fork a child process and exit.

    :return: nothing
    """
    if os.fork():
        # parent exits
        os._exit(0)
    os.setsid()


def _valid_module(path: str, file_name: str) -> bool:
    """
    Check if file is a valid python module.

    :param path: path to file
    :param file_name: file name to check
    :return: True if a valid python module file, False otherwise
    """
    file_path = os.path.join(path, file_name)
    if not os.path.isfile(file_path):
        return False

    if file_name.startswith("_"):
        return False

    if not file_name.endswith(".py"):
        return False

    return True


def _is_class(module: Any, member: Type, clazz: Type) -> bool:
    """
    Validates if a module member is a class and an instance of a CoreService.

    :param module: module to validate for service
    :param member: member to validate for service
    :param clazz: clazz type to check for validation
    :return: True if a valid service, False otherwise
    """
    if not inspect.isclass(member):
        return False

    if not issubclass(member, clazz):
        return False

    if member.__module__ != module.__name__:
        return False

    return True


def close_onexec(fd: int) -> None:
    """
    Close on execution of a shell process.

    :param fd: file descriptor to close
    :return: nothing
    """
    fdflags = fcntl.fcntl(fd, fcntl.F_GETFD)
    fcntl.fcntl(fd, fcntl.F_SETFD, fdflags | fcntl.FD_CLOEXEC)


def which(command: str, required: bool) -> str:
    """
    Find location of desired executable within current PATH.

    :param command: command to find location for
    :param required: command is required to be found, false otherwise
    :return: command location or None
    :raises ValueError: when not found and required
    """
    found_path = shutil.which(command)
    if found_path is None and required:
        raise ValueError(f"failed to find required executable({command}) in path")
    return found_path
{'fraction_non_alphanumeric': 0.08963337547408344, 'fraction_numerical': 0.0025284450063211127, 'mean_word_length': 2.9143988124690745, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2088907', 'n_tokens_mistral': 2448, 'n_tokens_neox': 2383, 'n_words': 864}