<commit_before># -*- coding: utf-8 -*-

import traceback
import time
from multiprocessing import Queue, Process
from sqlalchemy import update

from nailgun.api.models import Task
from nailgun.task.helpers import TaskHelper
from nailgun.logger import logger
from nailgun.db import make_session, make_engine

PLUGIN_PROCESSING_QUEUE = None


def get_queue():
    global PLUGIN_PROCESSING_QUEUE
    if not PLUGIN_PROCESSING_QUEUE:
        PLUGIN_PROCESSING_QUEUE = Queue()

    return PLUGIN_PROCESSING_QUEUE


class PluginProcessor(Process):
    """
    Separate process. When plugin added in the queue
    process started to processing plugin
    """
    def __init__(self):
        Process.__init__(self)
        self.db = make_session(make_engine())
        self.queue = get_queue()

        from nailgun.plugin.manager import PluginManager
        self.plugin_manager = PluginManager(self.db)

    def run(self):
        while True:
            task_uuid = None
            try:
                task_uuid = self.queue.get()
                self.plugin_manager.process(task_uuid)
            except Exception as exc:
                if task_uuid:
                    self.set_error(task_uuid, exc)
                logger.error(traceback.format_exc())
                time.sleep(1)

    def set_error(self, task_uuid, msg):
        self.db.query(Task).filter_by(uuid=task_uuid).update({
            'status': 'error',
            'progress': 100,
            'message': str(msg)})
        self.db.commit()
<commit_msg>Close db session on terminate<commit_after># -*- coding: utf-8 -*-

import traceback
import time
from multiprocessing import Queue, Process
from sqlalchemy import update

from nailgun.api.models import Task
from nailgun.task.helpers import TaskHelper
from nailgun.logger import logger
from nailgun.db import make_session, make_engine

PLUGIN_PROCESSING_QUEUE = None


def get_queue():
    global PLUGIN_PROCESSING_QUEUE
    if not PLUGIN_PROCESSING_QUEUE:
        PLUGIN_PROCESSING_QUEUE = Queue()

    return PLUGIN_PROCESSING_QUEUE


class PluginProcessor(Process):
    """
    Separate process. When plugin added in the queue
    process started to processing plugin
    """
    def __init__(self):
        Process.__init__(self)
        self.db = make_session(make_engine())
        self.queue = get_queue()

        from nailgun.plugin.manager import PluginManager
        self.plugin_manager = PluginManager(self.db)

    def run(self):
        while True:
            task_uuid = None
            try:
                task_uuid = self.queue.get()
                self.plugin_manager.process(task_uuid)
            except Exception as exc:
                if task_uuid:
                    self.set_error(task_uuid, exc)
                logger.error(traceback.format_exc())
                time.sleep(1)

    def set_error(self, task_uuid, msg):
        self.db.query(Task).filter_by(uuid=task_uuid).update({
            'status': 'error',
            'progress': 100,
            'message': str(msg)})
        self.db.commit()

    def terminate(self):
        self.db.close()
        super(PluginProcessor, self).terminate()

{'fraction_non_alphanumeric': 0.0853697238971755, 'fraction_numerical': 0.0031735956839098697, 'mean_word_length': 2.411255411255411, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7038033', 'n_tokens_mistral': 941, 'n_tokens_neox': 899, 'n_words': 221}