<commit_before>"""
Scrapy Shell

See documentation in docs/topics/shell.rst
"""

import signal

from twisted.internet import reactor, threads
from twisted.python.failure import Failure

from scrapy import log
from scrapy.item import BaseItem
from scrapy.spider import BaseSpider
from scrapy.selector import XPathSelector, XmlXPathSelector, HtmlXPathSelector
from scrapy.utils.spider import create_spider_for_request
from scrapy.utils.misc import load_object
from scrapy.utils.response import open_in_browser
from scrapy.utils.url import any_to_uri
from scrapy.utils.console import start_python_console
from scrapy.conf import settings
from scrapy.settings import Settings
from scrapy.http import Request, Response, TextResponse

class Shell(object):

    relevant_classes = (BaseSpider, Request, Response, BaseItem, \
        XPathSelector, Settings)

    def __init__(self, crawler, update_vars=None, inthread=False):
        self.crawler = crawler
        self.vars = {}
        self.update_vars = update_vars or (lambda x: None)
        self.item_class = load_object(settings['DEFAULT_ITEM_CLASS'])
        self.inthread = inthread

    def start(self, *a, **kw):
        # disable accidental Ctrl-C key press from shutting down the engine
        signal.signal(signal.SIGINT, signal.SIG_IGN)
        if self.inthread:
            return threads.deferToThread(self._start, *a, **kw)
        else:
            self._start(*a, **kw)

    def _start(self, url=None, request=None, response=None, spider=None):
        if url:
            self.fetch(url, spider)
        elif request:
            self.fetch(request, spider)
        elif response:
            request = response.request
            self.populate_vars(request.url, response, request, spider)
        start_python_console(self.vars)

    def _schedule(self, request, spider):
        if spider is None:
            spider = create_spider_for_request(self.crawler.spiders, request, \
                BaseSpider('default'), log_multiple=True)
        self.crawler.engine.open_spider(spider)
        return self.crawler.engine.schedule(request, spider)

    def fetch(self, request_or_url, spider=None):
        if isinstance(request_or_url, Request):
            request = request_or_url
            url = request.url
        else:
            url = any_to_uri(request_or_url)
            request = Request(url, dont_filter=True)
        response = None
        try:
            response = threads.blockingCallFromThread(reactor, \
                self._schedule, request, spider)
        except:
            log.err(Failure(), "Error fetching response", spider=spider)
        self.populate_vars(url, response, request, spider)

    def populate_vars(self, url=None, response=None, request=None, spider=None):
        item = self.item_class()
        self.vars['item'] = item
        self.vars['settings'] = settings
        if url:
            if isinstance(response, TextResponse):
                self.vars['xxs'] = XmlXPathSelector(response)
                self.vars['hxs'] = HtmlXPathSelector(response)
            self.vars['response'] = response
            self.vars['request'] = request
            self.vars['spider'] = spider
        if self.inthread:
            self.vars['fetch'] = self.fetch<commit_msg>Fixed unbounded spider error in shell, and enclosed fetch() method in a try/except block for logging errors more reliably<commit_after>"""
Scrapy Shell

See documentation in docs/topics/shell.rst
"""

import signal

from twisted.internet import reactor, threads
from twisted.python.failure import Failure

from scrapy import log
from scrapy.item import BaseItem
from scrapy.spider import BaseSpider
from scrapy.selector import XPathSelector, XmlXPathSelector, HtmlXPathSelector
from scrapy.utils.spider import create_spider_for_request
from scrapy.utils.misc import load_object
from scrapy.utils.response import open_in_browser
from scrapy.utils.url import any_to_uri
from scrapy.utils.console import start_python_console
from scrapy.conf import settings
from scrapy.settings import Settings
from scrapy.http import Request, Response, TextResponse

class Shell(object):

    relevant_classes = (BaseSpider, Request, Response, BaseItem, \
        XPathSelector, Settings)

    def __init__(self, crawler, update_vars=None, inthread=False):
        self.crawler = crawler
        self.vars = {}
        self.update_vars = update_vars or (lambda x: None)
        self.item_class = load_object(settings['DEFAULT_ITEM_CLASS'])
        self.inthread = inthread

    def start(self, *a, **kw):
        # disable accidental Ctrl-C key press from shutting down the engine
        signal.signal(signal.SIGINT, signal.SIG_IGN)
        if self.inthread:
            return threads.deferToThread(self._start, *a, **kw)
        else:
            self._start(*a, **kw)

    def _start(self, url=None, request=None, response=None, spider=None):
        if url:
            self.fetch(url, spider)
        elif request:
            self.fetch(request, spider)
        elif response:
            request = response.request
            self.populate_vars(request.url, response, request, spider)
        start_python_console(self.vars)

    def _schedule(self, request, spider):
        if spider is None:
            spider = create_spider_for_request(self.crawler.spiders, request, \
                BaseSpider('default'), log_multiple=True)
        spider.set_crawler(self.crawler)
        self.crawler.engine.open_spider(spider)
        return self.crawler.engine.schedule(request, spider)

    def fetch(self, request_or_url, spider=None):
        # we enclose all this code in a try/except block to see errors when
        # they happen in a thread
        try:
            if isinstance(request_or_url, Request):
                request = request_or_url
                url = request.url
            else:
                url = any_to_uri(request_or_url)
                request = Request(url, dont_filter=True)
            response = None
            response = threads.blockingCallFromThread(reactor, \
                self._schedule, request, spider)
            self.populate_vars(url, response, request, spider)
        except:
            log.err(Failure(), "Error fetching: %s" % request_or_url, spider=spider)

    def populate_vars(self, url=None, response=None, request=None, spider=None):
        item = self.item_class()
        self.vars['item'] = item
        self.vars['settings'] = settings
        if url:
            if isinstance(response, TextResponse):
                self.vars['xxs'] = XmlXPathSelector(response)
                self.vars['hxs'] = HtmlXPathSelector(response)
            self.vars['response'] = response
            self.vars['request'] = request
            self.vars['spider'] = spider
        if self.inthread:
            self.vars['fetch'] = self.fetch
{'fraction_non_alphanumeric': 0.0900218499635834, 'fraction_numerical': 0.0, 'mean_word_length': 2.735582154515778, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2433612', 'n_tokens_mistral': 1977, 'n_tokens_neox': 1893, 'n_words': 493}