<commit_before>"""blackhole.utils - A utility module used for
methods and features that do not belong in
their own module."""

import os
import sys
import grp
import pwd
import signal

from tornado.options import options
import setproctitle

from blackhole.log import log


def setgid():
    """
    Change our existing group.

    Used to drop from root privileges down to a less
    privileged group.

    MUST be called BEFORE setuid, not after.
    """
    try:
        os.setgid(grp.getgrnam(options.group).gr_gid)
    except KeyError:
        log.error("Group '%s' does not exist" % options.group)
        sys.exit(1)
    except OSError:
        log.error("You do not have permission to switch to group '%s'"
                  % options.group)
        sys.exit(1)


def setuid():
    """
    Change our existing user.

    Used to drop from root privileges down to a less
    privileged user

    MUST be called AFTER setgid, not before.
    """
    try:
        os.setuid(pwd.getpwnam(options.user).pw_uid)
    except KeyError:
        log.error("User '%s' does not exist" % options.user)
        sys.exit(1)
    except OSError:
        log.error("You do not have permission to switch to user '%s'"
                  % options.user)
        sys.exit(1)


def terminate(signum, frame):
    """
    Terminate the parent process and send signals
    to shut down it's children

    Iterates over the child pids in the frame
    and sends the SIGTERM signal to shut them
    down.
    """
    try:
        for pid in frame.f_locals['children'].iterkeys():
            os.kill(pid, signal.SIGTERM)
    except KeyError:
        # not the parent
        pass
    if os.path.exists(options.pid):
        os.remove(options.pid)
    sys.exit(0)

def set_process_title():
    """
    Set the title of the process.

    If the process is the master, set
    a master title, otherwise set
    worker.
    """
    log.info("test")
    if os.path.exists(options.pid):
        pid = int(file(options.pid, 'r').read().strip())
        log.info("%s and %s" % (pid, os.getpid()))
        if pid == os.getpid():<commit_msg>Remove test log message<commit_after>"""blackhole.utils - A utility module used for
methods and features that do not belong in
their own module."""

import os
import sys
import grp
import pwd
import signal

from tornado.options import options
import setproctitle

from blackhole.log import log


def setgid():
    """
    Change our existing group.

    Used to drop from root privileges down to a less
    privileged group.

    MUST be called BEFORE setuid, not after.
    """
    try:
        os.setgid(grp.getgrnam(options.group).gr_gid)
    except KeyError:
        log.error("Group '%s' does not exist" % options.group)
        sys.exit(1)
    except OSError:
        log.error("You do not have permission to switch to group '%s'"
                  % options.group)
        sys.exit(1)


def setuid():
    """
    Change our existing user.

    Used to drop from root privileges down to a less
    privileged user

    MUST be called AFTER setgid, not before.
    """
    try:
        os.setuid(pwd.getpwnam(options.user).pw_uid)
    except KeyError:
        log.error("User '%s' does not exist" % options.user)
        sys.exit(1)
    except OSError:
        log.error("You do not have permission to switch to user '%s'"
                  % options.user)
        sys.exit(1)


def terminate(signum, frame):
    """
    Terminate the parent process and send signals
    to shut down it's children

    Iterates over the child pids in the frame
    and sends the SIGTERM signal to shut them
    down.
    """
    try:
        for pid in frame.f_locals['children'].iterkeys():
            os.kill(pid, signal.SIGTERM)
    except KeyError:
        # not the parent
        pass
    if os.path.exists(options.pid):
        os.remove(options.pid)
    sys.exit(0)

def set_process_title():
    """
    Set the title of the process.

    If the process is the master, set
    a master title, otherwise set
    worker.
    """
    if os.path.exists(options.pid):
        pid = int(file(options.pid, 'r').read().strip())
        log.info("%s and %s" % (pid, os.getpid()))
        if pid == os.getpid():
{'fraction_non_alphanumeric': 0.10049893086243764, 'fraction_numerical': 0.0023758612497030173, 'mean_word_length': 2.304552590266876, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4508684', 'n_tokens_mistral': 1306, 'n_tokens_neox': 1263, 'n_words': 443}