<commit_before>import datetime
import logging
import multiprocessing
import os
import re
import time

from lobster.commands.plot import Plotter
from lobster.util import PartiallyMutable

logger = logging.getLogger('lobster.actions')

cmd_re = re.compile('^.* = [0-9]+$')

def runplots(plotter, foremen):
    try:
        plotter.make_plots(foremen=foremen)
    except Exception as e:
        logger.error("plotting failed with: {}".format(e))
        import traceback
        traceback.print_exc()

class DummyQueue(object):
    def start(*args):
        pass

    def put(*args):
        pass

    def get(*args):
        return None

class Actions(object):
    def __init__(self, config, source):
        fn = os.path.join(config.workdir, 'ipc')
        if not os.path.exists(fn):
            os.mkfifo(fn)
        self.fifo = os.fdopen(os.open(fn, os.O_RDONLY|os.O_NONBLOCK))
        self.config = config
        self.source = source

        if config.plotdir:
            logger.info('plots in {0} will be updated automatically'.format(config.plotdir))
            if config.foremen_logs:
                logger.info('foremen logs will be included from: {0}'.format(', '.join(config.foremen_logs)))
            self.plotter = Plotter(config)

        self.__last = datetime.datetime.now()

    def __communicate(self):
        cmds = map(str.strip, self.fifo.readlines())
        for cmd in cmds:
            logger.debug('received commands: {}'.format(cmd))
            if not cmd_re.match(cmd):
                logger.error('invalid command received: {}'.format(cmd))
                continue
            try:
                with PartiallyMutable.lockdown():
                    exec cmd in {'config': self.config}, {}
                    self.config.save()
            except Exception as e:
                logger.error('caught exeption from command: {}'.format(e))
        for component, attr, args in PartiallyMutable.changes():
            if attr is not None:
                logger.info('updating setup by calling {} of {} with {}'.format(attr, component, args))
                getattr(getattr(self, component), attr)(*args)

    def take(self, force=False):
        self.__communicate()

        now = datetime.datetime.now()
        if (now - self.__last).seconds > 15 * 60 or force:
            if not force and hasattr(self, 'p') and self.p.is_alive():
                logger.info('plotting still running, skipping')
            else:
                if force and hasattr(self, 'p'):
                    self.p.join()
                logger.info('starting plotting process')
                self.p = multiprocessing.Process(target=runplots, args=(self.plotter, self.config.foremen_logs))
                self.p.start()
            self.__last = now
<commit_msg>Always join plotting process.<commit_after>import datetime
import logging
import multiprocessing
import os
import re
import time

from lobster.commands.plot import Plotter
from lobster.util import PartiallyMutable

logger = logging.getLogger('lobster.actions')

cmd_re = re.compile('^.* = [0-9]+$')

def runplots(plotter, foremen):
    try:
        plotter.make_plots(foremen=foremen)
    except Exception as e:
        logger.error("plotting failed with: {}".format(e))
        import traceback
        traceback.print_exc()

class DummyQueue(object):
    def start(*args):
        pass

    def put(*args):
        pass

    def get(*args):
        return None

class Actions(object):
    def __init__(self, config, source):
        fn = os.path.join(config.workdir, 'ipc')
        if not os.path.exists(fn):
            os.mkfifo(fn)
        self.fifo = os.fdopen(os.open(fn, os.O_RDONLY|os.O_NONBLOCK))
        self.config = config
        self.source = source

        if config.plotdir:
            logger.info('plots in {0} will be updated automatically'.format(config.plotdir))
            if config.foremen_logs:
                logger.info('foremen logs will be included from: {0}'.format(', '.join(config.foremen_logs)))
            self.plotter = Plotter(config)

        self.__last = datetime.datetime.now()

    def __communicate(self):
        cmds = map(str.strip, self.fifo.readlines())
        for cmd in cmds:
            logger.debug('received commands: {}'.format(cmd))
            if not cmd_re.match(cmd):
                logger.error('invalid command received: {}'.format(cmd))
                continue
            try:
                with PartiallyMutable.lockdown():
                    exec cmd in {'config': self.config}, {}
                    self.config.save()
            except Exception as e:
                logger.error('caught exeption from command: {}'.format(e))
        for component, attr, args in PartiallyMutable.changes():
            if attr is not None:
                logger.info('updating setup by calling {} of {} with {}'.format(attr, component, args))
                getattr(getattr(self, component), attr)(*args)

    def take(self, force=False):
        self.__communicate()

        now = datetime.datetime.now()
        if (now - self.__last).seconds > 15 * 60 or force:
            if not force and hasattr(self, 'p') and self.p.is_alive():
                logger.info('plotting still running, skipping')
            else:
                if hasattr(self, 'p'):
                    logger.debug('joining plotting process')
                    self.p.join()
                logger.info('starting plotting process')
                self.p = multiprocessing.Process(target=runplots, args=(self.plotter, self.config.foremen_logs))
                self.p.start()
            self.__last = now

{'fraction_non_alphanumeric': 0.11653934979570084, 'fraction_numerical': 0.0028424231657488007, 'mean_word_length': 2.1019283746556474, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1830337', 'n_tokens_mistral': 1639, 'n_tokens_neox': 1558, 'n_words': 402}