<commit_before># Copyright (C) 2016 Cuckoo Foundation.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

import imp
import inspect
import logging
import multiprocessing
import os.path
import pkg_resources
import sys

try:
    import pwd
    HAVE_PWD = True
except ImportError:
    HAVE_PWD = False

import cuckoo

log = logging.getLogger(__name__)

# Cuckoo Working Directory base path.
_root = None
_raw = None

version = pkg_resources.require("Cuckoo")[0].version

def set_cwd(path, raw=None):
    global _root, _raw
    _root = path
    _raw = raw

def cwd(*args, **kwargs):
    """Returns absolute path to this file in the Cuckoo Working Directory or
    optionally - when private=True has been passed along - to our private
    Cuckoo Working Directory which is not configurable."""
    if kwargs.pop("private", False):
        return os.path.join(cuckoo.__path__[0], "data-private", *args)
    elif kwargs.pop("raw", False):
        return _raw
    else:
        return os.path.join(_root, *args)

def mkdir(*args):
    """Create a directory without throwing exceptions if it already exists."""
    dirpath = os.path.join(*args)
    if not os.path.isdir(dirpath):
        os.mkdir(dirpath)

def getuser():
    if HAVE_PWD:
        return pwd.getpwuid(os.getuid())[0]
    return ""

def load_signatures():
    """Loads additional Signatures from the Cuckoo Working Directory.

    This method is quite hacky in the sense that it magically imports
    Signatures from an arbitrary directory - one that doesn't belong to the
    Cuckoo package directly.

    Furthermore this method provides backwards compatibility with older
    Signatures which rely on the "lib.cuckoo.common.abstracts" import, one
    that can now be accessed as "cuckoo.common.abstracts".
    """
    # We need to create each module separately.
    module_names = (
        "lib", "lib.cuckoo", "lib.cuckoo.common",
        "lib.cuckoo.common.abstracts",
    )

    for module_name in module_names:
        sys.modules[module_name] = imp.new_module(module_name)

    # Import this here in order to avoid recursive import statements.
    from cuckoo.common.abstracts import Signature
    sys.modules["lib.cuckoo.common.abstracts"].Signature = Signature

    # Don't clobber the Cuckoo Working Directory with .pyc files.
    dont_write_bytecode = sys.dont_write_bytecode
    sys.dont_write_bytecode = True

    modules = {
        "android": dict(platform="android"),
        "cross": dict(),
        "darwin": dict(platform="darwin"),
        "network": dict(),
        "windows": dict(platform="windows"),
    }

    # Prepare a module for each signature directory.
    for modname in modules:
        module_name = "cuckoo.signatures.%s" % modname
        sys.modules[module_name] = imp.new_module(module_name)

    # Import each Signature that we find in the Cuckoo Working Directory.
    for dirpath, dirnames, filenames in os.walk(cwd("signatures")):
        for filename in filenames:
            if filename.endswith(".pyc") or filename.startswith("__init__"):
                continue

            # E.g., "cuckoo.signatures.network.network_http".
            category = os.path.basename(dirpath)
            module_name = "cuckoo.signatures.%s.%s" % (
                category, os.path.splitext(filename)[0]
            )
            module = imp.load_source(module_name, cwd(dirpath, filename))

            # Locate each Signature in this module and assign the
            # per-category attributes to it.
            for entry in module.__dict__.values():
                if not inspect.isclass(entry):
                    continue

                if not issubclass(entry, Signature) or entry == Signature:
                    continue

                for key, value in modules.get(category, {}).items():
                    setattr(entry, key, value)

    # Overwrite all Signatures that are in-place by all the Signatures that
    # have been registered at this point, literally.
    cuckoo.signatures.plugins[:] = Signature.__subclasses__()
    sys.dont_write_bytecode = dont_write_bytecode

def _worker(conn, func, *args, **kwargs):
    conn.send(func(*args, **kwargs))
    conn.close()

def dispatch(func, args=(), kwargs={}, timeout=60, process=True):
    """Dispatch a function call to a separate process or thread to execute with
    a maximum provided timeout. Note that in almost all occurrences a separate
    process should be used as otherwise we might end up with out-of-order
    locking mechanism instances, resulting in undefined behavior later on."""
    if not isinstance(args, tuple) or not isinstance(kwargs, dict):
        raise RuntimeError("args must be a tuple and kwargs a dict")

    if not process:
        raise RuntimeError("no support yet for dispatch(process=False)")

    parent, child = multiprocessing.Pipe()
    p = multiprocessing.Process(
        target=_worker, args=(child, func) + args, kwargs=kwargs
    )
    p.start()
    p.join(timeout)
    if p.is_alive():
        p.terminate()
        parent.close()
        return

    ret = parent.recv()
    parent.close()
    return ret<commit_msg>dispatch hangs indefinitely on windows if the pipe is duplex<commit_after># Copyright (C) 2016 Cuckoo Foundation.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

import imp
import inspect
import logging
import multiprocessing
import os.path
import pkg_resources
import sys

try:
    import pwd
    HAVE_PWD = True
except ImportError:
    HAVE_PWD = False

import cuckoo

log = logging.getLogger(__name__)

# Cuckoo Working Directory base path.
_root = None
_raw = None

version = pkg_resources.require("Cuckoo")[0].version

def set_cwd(path, raw=None):
    global _root, _raw
    _root = path
    _raw = raw

def cwd(*args, **kwargs):
    """Returns absolute path to this file in the Cuckoo Working Directory or
    optionally - when private=True has been passed along - to our private
    Cuckoo Working Directory which is not configurable."""
    if kwargs.pop("private", False):
        return os.path.join(cuckoo.__path__[0], "data-private", *args)
    elif kwargs.pop("raw", False):
        return _raw
    else:
        return os.path.join(_root, *args)

def mkdir(*args):
    """Create a directory without throwing exceptions if it already exists."""
    dirpath = os.path.join(*args)
    if not os.path.isdir(dirpath):
        os.mkdir(dirpath)

def getuser():
    if HAVE_PWD:
        return pwd.getpwuid(os.getuid())[0]
    return ""

def load_signatures():
    """Loads additional Signatures from the Cuckoo Working Directory.

    This method is quite hacky in the sense that it magically imports
    Signatures from an arbitrary directory - one that doesn't belong to the
    Cuckoo package directly.

    Furthermore this method provides backwards compatibility with older
    Signatures which rely on the "lib.cuckoo.common.abstracts" import, one
    that can now be accessed as "cuckoo.common.abstracts".
    """
    # We need to create each module separately.
    module_names = (
        "lib", "lib.cuckoo", "lib.cuckoo.common",
        "lib.cuckoo.common.abstracts",
    )

    for module_name in module_names:
        sys.modules[module_name] = imp.new_module(module_name)

    # Import this here in order to avoid recursive import statements.
    from cuckoo.common.abstracts import Signature
    sys.modules["lib.cuckoo.common.abstracts"].Signature = Signature

    # Don't clobber the Cuckoo Working Directory with .pyc files.
    dont_write_bytecode = sys.dont_write_bytecode
    sys.dont_write_bytecode = True

    modules = {
        "android": dict(platform="android"),
        "cross": dict(),
        "darwin": dict(platform="darwin"),
        "network": dict(),
        "windows": dict(platform="windows"),
    }

    # Prepare a module for each signature directory.
    for modname in modules:
        module_name = "cuckoo.signatures.%s" % modname
        sys.modules[module_name] = imp.new_module(module_name)

    # Import each Signature that we find in the Cuckoo Working Directory.
    for dirpath, dirnames, filenames in os.walk(cwd("signatures")):
        for filename in filenames:
            if filename.endswith(".pyc") or filename.startswith("__init__"):
                continue

            # E.g., "cuckoo.signatures.network.network_http".
            category = os.path.basename(dirpath)
            module_name = "cuckoo.signatures.%s.%s" % (
                category, os.path.splitext(filename)[0]
            )
            module = imp.load_source(module_name, cwd(dirpath, filename))

            # Locate each Signature in this module and assign the
            # per-category attributes to it.
            for entry in module.__dict__.values():
                if not inspect.isclass(entry):
                    continue

                if not issubclass(entry, Signature) or entry == Signature:
                    continue

                for key, value in modules.get(category, {}).items():
                    setattr(entry, key, value)

    # Overwrite all Signatures that are in-place by all the Signatures that
    # have been registered at this point, literally.
    cuckoo.signatures.plugins[:] = Signature.__subclasses__()
    sys.dont_write_bytecode = dont_write_bytecode

def _worker(conn, func, *args, **kwargs):
    conn.send(func(*args, **kwargs))
    conn.close()

def dispatch(func, args=(), kwargs={}, timeout=60, process=True):
    """Dispatch a function call to a separate process or thread to execute with
    a maximum provided timeout. Note that in almost all occurrences a separate
    process should be used as otherwise we might end up with out-of-order
    locking mechanism instances, resulting in undefined behavior later on."""
    if not isinstance(args, tuple) or not isinstance(kwargs, dict):
        raise RuntimeError("args must be a tuple and kwargs a dict")

    if not process:
        raise RuntimeError("no support yet for dispatch(process=False)")

    parent, child = multiprocessing.Pipe(duplex=False)
    p = multiprocessing.Process(
        target=_worker, args=(child, func) + args, kwargs=kwargs
    )
    p.start()
    p.join(timeout)
    if p.is_alive():
        p.terminate()
        parent.close()
        return

    ret = parent.recv()
    parent.close()
    return ret
{'fraction_non_alphanumeric': 0.09541582926595847, 'fraction_numerical': 0.0019140587616039812, 'mean_word_length': 2.853244837758112, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5061142', 'n_tokens_mistral': 3061, 'n_tokens_neox': 2960, 'n_words': 972}