<commit_before>import time
import sys
import subprocess
import logging

from nlpipe import client
from nlpipe.module import get_module

from multiprocessing import Process
from configparser import SafeConfigParser
from pydoc import locate

class Worker(Process):
    """
    Base class for NLP workers.
    """

    sleep_timeout = 1

    def __init__(self, client, module):
        """
        :param client: a Client object to connect to the NLP Server
        """
        super().__init__()
        self.client = client
        self.module = module

    def run(self):
        while True:
            id, doc = self.client.get_task(self.module.name)
            if id is None:
                time.sleep(self.sleep_timeout)
                continue
            logging.info("Received task {self.module.name}/{id} ({n} bytes)".format(n=len(doc), **locals()))
            try:
                result = self.module.process(doc)
                self.client.store_result(self.module.name, id, result)
                logging.debug("Succesfully completed task {self.module.name}/{id} ({n} bytes)"
                              .format(n=len(result), **locals()))
            except Exception as e:
                logging.exception("Exception on parsing {self.module.name}/{id}"
                              .format(**locals()))
                self.client.store_error(self.module.name, id, str(e))

<commit_msg>don't fail if you can't write error<commit_after>import time
import sys
import subprocess
import logging

from nlpipe import client
from nlpipe.module import get_module

from multiprocessing import Process
from configparser import SafeConfigParser
from pydoc import locate

class Worker(Process):
    """
    Base class for NLP workers.
    """

    sleep_timeout = 1

    def __init__(self, client, module):
        """
        :param client: a Client object to connect to the NLP Server
        """
        super().__init__()
        self.client = client
        self.module = module

    def run(self):
        while True:
            id, doc = self.client.get_task(self.module.name)
            if id is None:
                time.sleep(self.sleep_timeout)
                continue
            logging.info("Received task {self.module.name}/{id} ({n} bytes)".format(n=len(doc), **locals()))
            try:
                result = self.module.process(doc)
                self.client.store_result(self.module.name, id, result)
                logging.debug("Succesfully completed task {self.module.name}/{id} ({n} bytes)"
                              .format(n=len(result), **locals()))
            except Exception as e:
                logging.exception("Exception on parsing {self.module.name}/{id}"
                              .format(**locals()))
                try:
                    self.client.store_error(self.module.name, id, str(e))
                except:
                    logging.exception("Exception on storing error for {self.module.name}/{id}"
                                      .format(**locals()))


{'fraction_non_alphanumeric': 0.10738916256157635, 'fraction_numerical': 0.0006568144499178982, 'mean_word_length': 1.909264565425024, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5475994', 'n_tokens_mistral': 831, 'n_tokens_neox': 797, 'n_words': 219}