<commit_before>from multiprocessing import Process, Queue

import click
from scrapy import signals
from scrapy.crawler import CrawlerProcess
from pydispatch import dispatcher


class CrawlerWorker(Process):
    def __init__(
        self, settings, return_items, print_exception=True, *crawl_args, **crawl_kwargs
    ):
        super(CrawlerWorker, self).__init__()
        self.result_queue = Queue()
        self.crawl_args = crawl_args
        self.crawl_kwargs = crawl_kwargs
        self.print_exception = print_exception

        self.inner_exception = None

        self.process = CrawlerProcess(settings)
        self.items = []
        dispatcher.connect(self._spider_error, signals.spider_error)
        if return_items:
            dispatcher.connect(self._add_item, signals.item_scraped)

    def _add_item(self, item):
        self.items.append(item)

    def _spider_error(self, failure):
        self.result_queue.put(failure.value)
        self.result_queue.put(failure.getTraceback())

    def run(self):
        self.process.crawl(*self.crawl_args, **self.crawl_kwargs)
        self.process.start()
        if self.result_queue.empty():
            self.result_queue.put(self.items)

    def start(self):
        super(CrawlerWorker, self).start()

        result = self.result_queue.get()

        if isinstance(result, Exception):
            inner_exception = self.result_queue.get()
            if self.print_exception:
                click.echo("Error inside of Crawler Worker:")
                click.echo(inner_exception)
                click.echo("-------------------------------")
                click.echo("Error outside of Crawler Worker:")
            raise Exception(result)
        elif not result:
            return None
        else:
            return result
<commit_msg>Make sure you can exit crawler worker gracefully<commit_after>from multiprocessing import Process, Queue
import signal

import click
from scrapy import signals
from scrapy.crawler import CrawlerProcess
from pydispatch import dispatcher


class CrawlerWorker(Process):
    def __init__(
        self, settings, return_items, print_exception=True, *crawl_args, **crawl_kwargs
    ):
        super(CrawlerWorker, self).__init__()
        self.result_queue = Queue()
        self.crawl_args = crawl_args
        self.crawl_kwargs = crawl_kwargs
        self.print_exception = print_exception

        self.inner_exception = None

        self.process = CrawlerProcess(settings)
        self.kill_process = False
        self.items = []
        dispatcher.connect(self._spider_error, signals.spider_error)
        signal.signal(signal.SIGINT, self._exit_gracefully)
        signal.signal(signal.SIGTERM, self._exit_gracefully)
        if return_items:
            dispatcher.connect(self._add_item, signals.item_scraped)

    def _add_item(self, item):
        self.items.append(item)

    def _spider_error(self, failure):
        self.result_queue.put(failure.value)
        self.result_queue.put(failure.getTraceback())

    def _exit_gracefully(self, signum, frame):
        self.kill_process = True

    def run(self):
        self.process.crawl(*self.crawl_args, **self.crawl_kwargs)
        self.process.start()
        if self.result_queue.empty():
            self.result_queue.put(self.items)

    def start(self):
        super(CrawlerWorker, self).start()

        result = self.result_queue.get()

        if self.kill_process:
            raise KeyboardInterrupt
        if isinstance(result, Exception):
            inner_exception = self.result_queue.get()
            if self.print_exception:
                click.echo("Error inside of Crawler Worker:")
                click.echo(inner_exception)
                click.echo("-------------------------------")
                click.echo("Error outside of Crawler Worker:")
            raise Exception(result)
        elif not result:
            return None
        else:
            return result

{'fraction_non_alphanumeric': 0.10148577184588264, 'fraction_numerical': 0.0, 'mean_word_length': 2.3434343434343434, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2743727', 'n_tokens_mistral': 1140, 'n_tokens_neox': 1092, 'n_words': 229}