<commit_before>from __future__ import absolute_import
from six.moves.urllib.parse import urlsplit, urlunsplit

import formasaurus
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.settings import Settings
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.exceptions import CloseSpider

from .app import app, db
from .login_keychain import get_domain


settings = Settings(values=dict(
    ROBOTSTXT_OBEY = False,
    DEPTH_LIMIT = 3,
    DOWNLOAD_DELAY = 2.0,
    DEPTH_PRIORITY = 1,
    CONCURRENT_REQUESTS = 2,
    CONCURRENT_REQUESTS_PER_DOMAIN = 2,
    SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue',
    SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue',
    ))
configure_logging({'LOG_FORMAT': '%(levelname)s: %(message)s'})
crawl_runner = CrawlerRunner(settings)


class Spider(scrapy.Spider):
    name = 'spider'
    priority_patterns = [
        # Login links
        'login',
        'logon',
        'signin',
        'sign in',
        # Registration links
        'signup',
        'sign up',
        'register',
        'registration',
        'join',
    ]

    def __init__(self, url, credentials, *args, **kwargs):
        self.credentials = credentials
        self.start_urls = [url]
        self.link_extractor = LinkExtractor(allow_domains=[get_domain(url)])<commit_msg>Add another common registration marker<commit_after>from __future__ import absolute_import
from six.moves.urllib.parse import urlsplit, urlunsplit

import formasaurus
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.settings import Settings
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.exceptions import CloseSpider

from .app import app, db
from .login_keychain import get_domain


settings = Settings(values=dict(
    ROBOTSTXT_OBEY = False,
    DEPTH_LIMIT = 3,
    DOWNLOAD_DELAY = 2.0,
    DEPTH_PRIORITY = 1,
    CONCURRENT_REQUESTS = 2,
    CONCURRENT_REQUESTS_PER_DOMAIN = 2,
    SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue',
    SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue',
    ))
configure_logging({'LOG_FORMAT': '%(levelname)s: %(message)s'})
crawl_runner = CrawlerRunner(settings)


class Spider(scrapy.Spider):
    name = 'spider'
    priority_patterns = [
        # Login links
        'login',
        'logon',
        'signin',
        'sign in',
        # Registration links
        'signup',
        'sign up',
        'register',
        'registration',
        'account',
        'join',
    ]

    def __init__(self, url, credentials, *args, **kwargs):
        self.credentials = credentials
        self.start_urls = [url]
        self.link_extractor = LinkExtractor(allow_domains=[get_domain(url)])
{'fraction_non_alphanumeric': 0.09339488636363637, 'fraction_numerical': 0.004261363636363636, 'mean_word_length': 3.4856687898089174, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4967909', 'n_tokens_mistral': 948, 'n_tokens_neox': 913, 'n_words': 198}