<commit_before># Copyright (C) 2007-2010 <NAME>.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
import os, logging
from collections             import defaultdict
from threading               import Lock
from Exscript.util.decorator import bind

class Service(object):
    def __init__(self,
                 daemon,
                 name,
                 cfg_dir,
                 queue = None):
        self.cfg_dir   = cfg_dir
        self.daemon    = daemon
        self.name      = name
        self.queue     = queue
        self.task_lock = Lock()
        self.tasks     = defaultdict(list)  # Maps order ids to lists of tasks.

    def log(self, order, message):
        self.daemon.log(order, message)

    def get_logname(self, order, name = ''):
        return os.path.join(self.daemon.get_logdir(),
                            self.name,
                            str(order.get_id()),
                            name)

    def create_logger(self, order, name, level = logging.INFO):
        logfile   = self.get_logname(order, name)
        logger    = logging.getLogger(logfile)
        handler   = logging.FileHandler(logfile)
        formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
        logger.setLevel(logging.INFO)
        handler.setFormatter(formatter)
        logger.addHandler(handler)
        return logger

    def _update_host_logname(self, order, host):
        host.set_logname(self.get_logname(order, host.get_logname()))

    def config_file(self, name):
        return os.path.join(self.cfg_dir, name)

    def _track_task(self, order, task):
        with self.task_lock:
            task.signal_connect('done', self._task_done, order, task)
            self.tasks[order.id].append(task)

    def _task_done(self, order, task):
        with self.task_lock:
            self.tasks[order.id].remove(task)
            if not self.tasks[order.id]:
                del self.tasks[order.id]
                self.done(order)

    def enqueue_hosts(self, order, hosts, callback):
        hosts = order.get_hosts()
        for host in hosts:
            self._update_host_logname(order, host)

        # For performance reasons, we defer the starting of further
        # threads by pausing the queue.
        # We also need to pause to avoid getting a 'done' signal before
        # the signal is connected.
        self.queue.workqueue.pause()
        task = self.queue.run(hosts, callback)
        self._track_task(order, task)
        self.queue.workqueue.unpause()
        return task

    def enqueue(self, order, function, name):
        name = self.get_logname(order, name)
        self.queue.workqueue.pause()
        task = self.queue.enqueue(function, name)
        self._track_task(order, task)
        self.queue.workqueue.unpause()
        return task

    def check(self, order):
        return True

    def enter(self, order):
        return True

    def done(self, order):
        self.daemon.set_order_status_done(order)

    def set_order_status(self, order, status):
        self.daemon.set_order_status(order, status)

    def save_order(self, order):
        self.daemon.save_order(order)<commit_msg>exscriptd: fix: destroy loggers after an order completes.<commit_after># Copyright (C) 2007-2010 <NAME>.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
import os, logging
from collections             import defaultdict
from threading               import Lock
from Exscript.util.decorator import bind

class Service(object):
    def __init__(self,
                 daemon,
                 name,
                 cfg_dir,
                 queue = None):
        self.cfg_dir   = cfg_dir
        self.daemon    = daemon
        self.name      = name
        self.queue     = queue
        self.task_lock = Lock()
        self.tasks     = defaultdict(list) # Map order ids to lists of tasks.
        self.loggers   = defaultdict(list) # Map order ids to loggers.

    def log(self, order, message):
        self.daemon.log(order, message)

    def get_logname(self, order, name = ''):
        return os.path.join(self.daemon.get_logdir(),
                            self.name,
                            str(order.get_id()),
                            name)

    def create_logger(self, order, name, level = logging.INFO):
        logfile   = self.get_logname(order, name)
        logger    = logging.getLogger(logfile)
        handler   = logging.FileHandler(logfile)
        formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
        logger.setLevel(logging.INFO)
        handler.setFormatter(formatter)
        logger.addHandler(handler)
        self.loggers[order.id].append(logger)
        return logger

    def _free_loggers(self, order):
        for logger in self.loggers[order.id]:
            del logger.manager.loggerDict[logger.name]
            logger.manager = None

    def _update_host_logname(self, order, host):
        host.set_logname(self.get_logname(order, host.get_logname()))

    def config_file(self, name):
        return os.path.join(self.cfg_dir, name)

    def _track_task(self, order, task):
        with self.task_lock:
            task.signal_connect('done', self._task_done, order, task)
            self.tasks[order.id].append(task)

    def _task_done(self, order, task):
        with self.task_lock:
            self.tasks[order.id].remove(task)
            if not self.tasks[order.id]:
                del self.tasks[order.id]
                self.done(order)

    def enqueue_hosts(self, order, hosts, callback):
        hosts = order.get_hosts()
        for host in hosts:
            self._update_host_logname(order, host)

        # For performance reasons, we defer the starting of further
        # threads by pausing the queue.
        # We also need to pause to avoid getting a 'done' signal before
        # the signal is connected.
        self.queue.workqueue.pause()
        task = self.queue.run(hosts, callback)
        self._track_task(order, task)
        self.queue.workqueue.unpause()
        return task

    def enqueue(self, order, function, name):
        name = self.get_logname(order, name)
        self.queue.workqueue.pause()
        task = self.queue.enqueue(function, name)
        self._track_task(order, task)
        self.queue.workqueue.unpause()
        return task

    def check(self, order):
        return True

    def enter(self, order):
        return True

    def done(self, order):
        self._free_loggers(order)
        self.daemon.set_order_status_done(order)

    def set_order_status(self, order, status):
        self.daemon.set_order_status(order, status)

    def save_order(self, order):
        self.daemon.save_order(order)
{'fraction_non_alphanumeric': 0.08994307400379507, 'fraction_numerical': 0.0058191018342821, 'mean_word_length': 2.28595178719867, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5491188', 'n_tokens_mistral': 2284, 'n_tokens_neox': 2197, 'n_words': 658}