<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import

import logging
import gevent
from gevent import Greenlet, GreenletExit, sleep
from gevent.subprocess import Popen
from datetime import datetime, timedelta
import dateutil.tz
from os import setgid, setuid, setgroups, chdir, environ, getpid, setsid, killpg, getpgid
from pwd import getpwnam
from grp import getgrnam
from locale import getpreferredencoding
from subprocess import PIPE
import sys
import signal

from moneta import json
from moneta.http.client import HTTPClient
from moneta.http.http import HTTPRequest, parse_host_port

logger = logging.getLogger('moneta.watcher')

class MonetaWatcher(object):
    """ Watch a process and send updates back to the manager. """

    def __init__(self, nodename):
        """ Constructor """

        self.task = None
        self.taskconfig = {}
        self.processid = None
        self.manager = None
        self.nodename = nodename

        self.report = {}
        self.exec_greenlet = None
        self.notify_greenlet = None
        self.last_notification = datetime.utcnow().replace(tzinfo = dateutil.tz.tzutc())
        self.disable_notify = False

        self.notify_delay = 1
        self.giveup_notify_after = 10

    @staticmethod
    def spawn(processid, task, taskconfig, manager):
        """ Spawn a new watcher in another process """

        path = list(sys.argv)
        path.append('--watcher')

        process = Popen(args = path, stdin = PIPE)
        process.stdin.write(json.dumps({
            "manager": manager,
            "task": task,
            "processid": processid,
            "taskconfig": taskconfig
        }))
        process.stdin.close()
        logger.debug('Process %s (task %s) spawned and configured.', processid, task)

    def run(self):
        """ Run in Watcher Mode """
<commit_msg>Put the watcher in its own process group so upstart and the like will not kill it<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import

import logging
import gevent
from gevent import Greenlet, GreenletExit, sleep
from gevent.subprocess import Popen
from datetime import datetime, timedelta
import dateutil.tz
from os import setgid, setuid, setgroups, chdir, environ, getpid, setsid, killpg, getpgid
from pwd import getpwnam
from grp import getgrnam
from locale import getpreferredencoding
from subprocess import PIPE
import sys
import signal

from moneta import json
from moneta.http.client import HTTPClient
from moneta.http.http import HTTPRequest, parse_host_port

logger = logging.getLogger('moneta.watcher')

class MonetaWatcher(object):
    """ Watch a process and send updates back to the manager. """

    def __init__(self, nodename):
        """ Constructor """

        self.task = None
        self.taskconfig = {}
        self.processid = None
        self.manager = None
        self.nodename = nodename

        self.report = {}
        self.exec_greenlet = None
        self.notify_greenlet = None
        self.last_notification = datetime.utcnow().replace(tzinfo = dateutil.tz.tzutc())
        self.disable_notify = False

        self.notify_delay = 1
        self.giveup_notify_after = 10

    @staticmethod
    def spawn(processid, task, taskconfig, manager):
        """ Spawn a new watcher in another process """

        path = list(sys.argv)
        path.append('--watcher')

        process = Popen(args = path, stdin = PIPE, preexec_fn = setsid)
        process.stdin.write(json.dumps({
            "manager": manager,
            "task": task,
            "processid": processid,
            "taskconfig": taskconfig
        }))
        process.stdin.close()
        logger.debug('Process %s (task %s) spawned and configured.', processid, task)

    def run(self):
        """ Run in Watcher Mode """

{'fraction_non_alphanumeric': 0.09221902017291066, 'fraction_numerical': 0.002095886822111606, 'mean_word_length': 2.8256513026052104, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4073006', 'n_tokens_mistral': 1105, 'n_tokens_neox': 1074, 'n_words': 338}