<commit_before>from scrapy.command import ScrapyCommand
from scrapy.core.manager import scrapymanager
from scrapy.conf import settings
from scrapy.utils.url import is_url
from scrapy.spider import spiders
from scrapy.http import Request
from scrapy import log


class Command(ScrapyCommand):

    requires_project = True

    def syntax(self):
        return "[options] <domain|url> ..."

    def short_desc(self):
        return "Start crawling a domain or URL"

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("--nopipeline", dest="nopipeline", action="store_true", help="disable scraped item pipeline")
        parser.add_option("--restrict", dest="restrict", action="store_true", help="restrict crawling only to the given urls")
        parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", help="don't follow links (for use with URLs only)")
        parser.add_option("-c", "--callback", dest="callback", action="store", help="use the provided callback for starting to crawl the given url")

    def process_options(self, args, opts):
        ScrapyCommand.process_options(self, args, opts)
        if opts.nopipeline:
            settings.overrides['ITEM_PIPELINES'] = []

        if opts.restrict:
            settings.overrides['RESTRICT_TO_URLS'] = args

        if opts.nofollow:
            settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False

    def run(self, args, opts):
        if opts.callback:
            requests = []
            for a in args:
                if is_url(a):
                    spider = spiders.fromurl(a)
                    urls = [a]
                else:
                    spider = spiders.fromdomain(a)
                    urls = spider.start_urls if hasattr(spider.start_urls, '__iter__') else [spider.start_urls]

                if spider:
                    if hasattr(spider, opts.callback):
                        requests.extend(Request(url=url, callback=getattr(spider, opts.callback)) for url in urls)
                    else:
                        log.msg('Callback %s doesnt exist in spider %s' % (opts.callback, spider.domain_name), log.ERROR)
                else:
                    log.msg('Cannot find spider for %s' % a, log.ERROR)

            if requests:
                args = requests
            else:
                log.msg('Cannot create any requests with the provided arguments', log.ERROR)
                return

        scrapymanager.runonce(*args)<commit_msg>some line wrapping at 80 cols<commit_after>from scrapy.command import ScrapyCommand
from scrapy.core.manager import scrapymanager
from scrapy.conf import settings
from scrapy.utils.url import is_url
from scrapy.spider import spiders
from scrapy.http import Request
from scrapy import log


class Command(ScrapyCommand):

    requires_project = True

    def syntax(self):
        return "[options] <domain|url> ..."

    def short_desc(self):
        return "Start crawling a domain or URL"

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("--nopipeline", dest="nopipeline", action="store_true", \
            help="disable scraped item pipeline")
        parser.add_option("--restrict", dest="restrict", action="store_true", \
            help="restrict crawling only to the given urls")
        parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", \
            help="don't follow links (for use with URLs only)")
        parser.add_option("-c", "--callback", dest="callback", action="store", \
            help="use the provided callback for starting to crawl the given url")

    def process_options(self, args, opts):
        ScrapyCommand.process_options(self, args, opts)
        if opts.nopipeline:
            settings.overrides['ITEM_PIPELINES'] = []

        if opts.restrict:
            settings.overrides['RESTRICT_TO_URLS'] = args

        if opts.nofollow:
            settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False

    def run(self, args, opts):
        if opts.callback:
            requests = []
            for a in args:
                if is_url(a):
                    spider = spiders.fromurl(a)
                    urls = [a]
                else:
                    spider = spiders.fromdomain(a)
                    urls = spider.start_urls if hasattr(spider.start_urls, \
                        '__iter__') else [spider.start_urls]

                if spider:
                    if hasattr(spider, opts.callback):
                        requests.extend(Request(url=url, callback=getattr(spider, \
                            opts.callback)) for url in urls)
                    else:
                        log.msg('Callback %s doesnt exist in spider %s' % \
                            (opts.callback, spider.domain_name), log.ERROR)
                else:
                    log.msg('Cannot find spider for %s' % a, log.ERROR)

            if requests:
                args = requests
            else:
                log.msg('Cannot create any requests with the provided arguments', \
                    log.ERROR)
                return

        scrapymanager.runonce(*args)
{'fraction_non_alphanumeric': 0.10053516819571866, 'fraction_numerical': 0.00038226299694189603, 'mean_word_length': 2.03538283062645, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5694801', 'n_tokens_mistral': 1478, 'n_tokens_neox': 1385, 'n_words': 378}