<commit_before>"""
Extensions for debugging Scrapy

See documentation in docs/topics/extensions.rst
"""

import os
import sys
import signal
import traceback
import threading
from pdb import Pdb

from scrapy.utils.engine import format_engine_status
from scrapy import log


class StackTraceDump(object):

    def __init__(self, crawler=None):
        self.crawler = crawler
        try:
            signal.signal(signal.SIGUSR2, self.dump_stacktrace)
            signal.signal(signal.SIGQUIT, self.dump_stacktrace)
        except AttributeError:
            # win32 platforms don't support SIGUSR signals
            pass


    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler)

    def dump_stacktrace(self, signum, frame):
        stackdumps = self._thread_stacks()
        enginestatus = format_engine_status(self.crawler.engine)
        msg = "Dumping stack trace and engine status" \
            "\n{0}\n{1}".format(enginestatus, stackdumps)
        log.msg(msg)

    def _thread_stacks(self):
        id2name = dict((th.ident, th.name) for th in threading.enumerate())
        dumps = ''
        for id_, frame in sys._current_frames().items():
            name = id2name.get(id_, '')
            dump = ''.join(traceback.format_stack(frame))
            dumps += "# Thread: {0}({1})\n{2}\n".format(name, id_, dump)
        return dumps


class Debugger(object):
    def __init__(self):
        try:
            signal.signal(signal.SIGUSR2, self._enter_debugger)
        except AttributeError:<commit_msg>also dump (scrapy.utils.trackref) live references in StackTraceDump extension<commit_after>"""
Extensions for debugging Scrapy

See documentation in docs/topics/extensions.rst
"""

import sys
import signal
import traceback
import threading
from pdb import Pdb

from scrapy.utils.engine import format_engine_status
from scrapy.utils.trackref import format_live_refs
from scrapy import log


class StackTraceDump(object):

    def __init__(self, crawler=None):
        self.crawler = crawler
        self.dumprefs = crawler.settings.getbool('TRACK_REFS')
        try:
            signal.signal(signal.SIGUSR2, self.dump_stacktrace)
            signal.signal(signal.SIGQUIT, self.dump_stacktrace)
        except AttributeError:
            # win32 platforms don't support SIGUSR signals
            pass

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler)

    def dump_stacktrace(self, signum, frame):
        stackdumps = self._thread_stacks()
        enginestatus = format_engine_status(self.crawler.engine)
        liverefs = format_live_refs() if self.dumprefs else ""
        msg = "Dumping stack trace and engine status" \
            "\n{0}\n{1}\n{2}".format(enginestatus, liverefs, stackdumps)
        log.msg(msg)

    def _thread_stacks(self):
        id2name = dict((th.ident, th.name) for th in threading.enumerate())
        dumps = ''
        for id_, frame in sys._current_frames().items():
            name = id2name.get(id_, '')
            dump = ''.join(traceback.format_stack(frame))
            dumps += "# Thread: {0}({1})\n{2}\n".format(name, id_, dump)
        return dumps


class Debugger(object):
    def __init__(self):
        try:
            signal.signal(signal.SIGUSR2, self._enter_debugger)
        except AttributeError:
{'fraction_non_alphanumeric': 0.1011743450767841, 'fraction_numerical': 0.006925624811803674, 'mean_word_length': 2.7965714285714287, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6924489', 'n_tokens_mistral': 1033, 'n_tokens_neox': 1000, 'n_words': 238}