<commit_before>from hitchserve.service_handle import ServiceHandle
from hitchserve.service_engine import ServiceEngine
from hitchserve.hitch_exception import ServiceStartupTimeoutException
from colorama import Fore, Back, Style
import functools
import termios
import signal
import psutil
import time
import pyuv
import sys
import os

# TODO: Report actual directory being used at the beginning of the logs.
# TODO: Move all from-previous-test-run-process killing code here.
# TODO: Rename TestEngine -> OrchestrationEngine

class TestEngine(object):
    """Engine (and thread) that brings together all I/O and events driving hitchserve."""
    _timedout = False
    _ready = False
    _driver_sent_shutdown_signal = False
    _shutdown_triggered = False
    ipython_on = False

    def __init__(self, service_bundle):
        self.service_bundle = service_bundle

    def signal_cb(self, handle, signum):
        """Handle ctrl-C if not in ipython shell. Always shutdown on SIGTERM."""
        SIGNALS_TO_NAMES_DICT = dict((getattr(signal, n), n) \
            for n in dir(signal) if n.startswith('SIG') and '_' not in n )
        self.logline("Signal {} received...".format(SIGNALS_TO_NAMES_DICT.get(signum)))
        if signum in (signal.SIGTERM, signal.SIGHUP, signal.SIGQUIT, signal.SIGINT):
            self.stop()

    def handle_tail(self, fullfilename, title, color, handle, filename, events, error):
        with open(fullfilename, 'r') as filehandle:
            filehandle.seek(self.tail_positions[fullfilename])
            tailportion = filehandle.read()
        self.tail_positions[fullfilename] = self.tail_positions[fullfilename] + len(tailportion) #os.stat(fullfilename).st_size

        for line in tailportion.split('\n'):
            if line != "":
                if color is None:
                    self.logline(line, title=title)
                else:
                    self.logline(line, title=title, color=color)


    # TODO: Close tail handles after set up / poststart is finished.
    def initiate_tail(self, stdlog, title, color):
        filename = self.service_bundle.hitch_dir.stdlogdir() + os.sep + stdlog
        with open(filename, "a"):
            os.utime(filename, None)
        self.tail_positions[filename] = 0
        tail = pyuv.fs.FSEvent(self.loop)
        tail.start(filename, 0, functools.partial(self.handle_tail, filename, title, color))
        self.tail_handles.append(tail)
        with open(filename, "a"):
            os.utime(filename, None)

    def start(self, messages_to_driver, messages_to_bundle_engine):
        """Orchestrate processes and I/O pipes."""

        os.setpgrp()
        self.service_bundle.hitch_dir.save_pgid("bundleengine", os.getpgid(os.getpid()))
        self.service_engine = ServiceEngine(self, self.service_bundle.values())

        self.start_time = time.time()
        self.messages_to_driver = messages_to_driver
        self.messages_to_bundle_engine = messages_to_bundle_engine

        os.chdir(self.service_bundle.hitch_dir.project_directory)
        self.logfile = open(self.service_bundle.hitch_dir.testlog(), "a")

        self.loop = pyuv.Loop.default_loop()

        self.pipe_logfile = pyuv.Pipe(self.loop)
        self.pipe_logfile.open(self.logfile.fileno())

        if not self.service_bundle.quiet:
            self.pipe_stdout = pyuv.Pipe(self.loop)
            self.pipe_stdout.open(sys.stdout.fileno())

        self.service_engine.start_services_without_prerequisites()


        self.timer_handler = pyuv.Timer(self.loop)
        self.timer_handler.start(self.poll_handler, 0.01, 0.01)

        self.tail_handles = []
        self.tail_positions = {}

        self.initiate_tail("driver.out", "Test", None)
        self.initiate_tail("driver.err", "Err Test", Fore.YELLOW)

        self.service_engine.tail_setup_and_poststart()

        self.signal_h = pyuv.Signal(self.loop)
        self.signal_h.start(self.signal_cb, signal.SIGINT)
        self.signal_h.start(self.signal_cb, signal.SIGTERM)

        self.loop.run()

    def _close_pipes(self):
        """Close all the pipes in order to shut the engine down."""
        if not self.service_bundle.quiet:
            if not self.pipe_stdout.closed:
                self.pipe_stdout.close()
        for pipe in self.service_engine.pipes():
            if pipe is not None and not pipe.closed:
                pipe.close()
        if not self.signal_h.closed:
            self.signal_h.close()
        if not self.timer_handler.closed:
            self.timer_handler.close()<commit_msg>BUG : Orchestration engine process sometimes refuses to die after it is finished.<commit_after>from hitchserve.service_handle import ServiceHandle
from hitchserve.service_engine import ServiceEngine
from hitchserve.hitch_exception import ServiceStartupTimeoutException
from colorama import Fore, Back, Style
import functools
import termios
import signal
import psutil
import time
import pyuv
import sys
import os

# TODO: Report actual directory being used at the beginning of the logs.
# TODO: Move all from-previous-test-run-process killing code here.
# TODO: Rename TestEngine -> OrchestrationEngine

class TestEngine(object):
    """Engine (and thread) that brings together all I/O and events driving hitchserve."""
    _timedout = False
    _ready = False
    _driver_sent_shutdown_signal = False
    _shutdown_triggered = False
    ipython_on = False

    def __init__(self, service_bundle):
        self.service_bundle = service_bundle

    def signal_cb(self, handle, signum):
        """Handle ctrl-C if not in ipython shell. Always shutdown on SIGTERM."""
        SIGNALS_TO_NAMES_DICT = dict((getattr(signal, n), n) \
            for n in dir(signal) if n.startswith('SIG') and '_' not in n )
        self.logline("Signal {} received...".format(SIGNALS_TO_NAMES_DICT.get(signum)))
        if signum in (signal.SIGTERM, signal.SIGHUP, signal.SIGQUIT, signal.SIGINT):
            self.stop()

    def handle_tail(self, fullfilename, title, color, handle, filename, events, error):
        with open(fullfilename, 'r') as filehandle:
            filehandle.seek(self.tail_positions[fullfilename])
            tailportion = filehandle.read()
        self.tail_positions[fullfilename] = self.tail_positions[fullfilename] + len(tailportion) #os.stat(fullfilename).st_size

        for line in tailportion.split('\n'):
            if line != "":
                if color is None:
                    self.logline(line, title=title)
                else:
                    self.logline(line, title=title, color=color)


    # TODO: Close tail handles after set up / poststart is finished.
    def initiate_tail(self, stdlog, title, color):
        filename = self.service_bundle.hitch_dir.stdlogdir() + os.sep + stdlog
        with open(filename, "a"):
            os.utime(filename, None)
        self.tail_positions[filename] = 0
        tail = pyuv.fs.FSEvent(self.loop)
        tail.start(filename, 0, functools.partial(self.handle_tail, filename, title, color))
        self.tail_handles.append(tail)
        with open(filename, "a"):
            os.utime(filename, None)

    def start(self, messages_to_driver, messages_to_bundle_engine):
        """Orchestrate processes and I/O pipes."""

        os.setpgrp()
        self.service_bundle.hitch_dir.save_pgid("bundleengine", os.getpgid(os.getpid()))
        self.service_engine = ServiceEngine(self, self.service_bundle.values())

        self.start_time = time.time()
        self.messages_to_driver = messages_to_driver
        self.messages_to_bundle_engine = messages_to_bundle_engine

        os.chdir(self.service_bundle.hitch_dir.project_directory)
        self.logfile = open(self.service_bundle.hitch_dir.testlog(), "a")

        self.loop = pyuv.Loop.default_loop()

        self.pipe_logfile = pyuv.Pipe(self.loop)
        self.pipe_logfile.open(self.logfile.fileno())

        if not self.service_bundle.quiet:
            self.pipe_stdout = pyuv.Pipe(self.loop)
            self.pipe_stdout.open(sys.stdout.fileno())

        self.service_engine.start_services_without_prerequisites()


        self.timer_handler = pyuv.Timer(self.loop)
        self.timer_handler.start(self.poll_handler, 0.01, 0.01)

        self.tail_handles = []
        self.tail_positions = {}

        self.initiate_tail("driver.out", "Test", None)
        self.initiate_tail("driver.err", "Err Test", Fore.YELLOW)

        self.service_engine.tail_setup_and_poststart()

        self.signal_h = pyuv.Signal(self.loop)
        self.signal_h.start(self.signal_cb, signal.SIGINT)
        self.signal_h.start(self.signal_cb, signal.SIGTERM)

        self.loop.run()
        self.loop = None
        os.kill(os.getpid(), signal.SIGKILL)

    def _close_pipes(self):
        """Close all the pipes in order to shut the engine down."""
        if not self.service_bundle.quiet:
            if not self.pipe_stdout.closed:
                self.pipe_stdout.close()
        for pipe in self.service_engine.pipes():
            if pipe is not None and not pipe.closed:
                pipe.close()
        if not self.signal_h.closed:
            self.signal_h.close()
        if not self.timer_handler.closed:
            self.timer_handler.close()
{'fraction_non_alphanumeric': 0.10151515151515152, 'fraction_numerical': 0.0017316017316017316, 'mean_word_length': 2.8265010351966873, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '87313', 'n_tokens_mistral': 2844, 'n_tokens_neox': 2742, 'n_words': 618}