<commit_before>"""
Global TODO:
* make task_%s_preprocess methods
"""
from __future__ import absolute_import
import types
import signal
import inspect
import traceback
import logging
from collections import defaultdict
import os
import time
import json
import cPickle as pickle
import anydbm
import multiprocessing
import zlib
from hashlib import sha1
from urlparse import urljoin
from random import randint

import Queue
from ..base import GLOBAL_STATE, Grab
from .error import SpiderError, SpiderMisuseError, FatalError
from .task import Task
from .data import Data
from .pattern import SpiderPattern
from .stat  import SpiderStat
from .transport.multicurl import MulticurlTransport
from ..proxylist import ProxyList

DEFAULT_TASK_PRIORITY = 100
RANDOM_TASK_PRIORITY_RANGE = (50, 100)
TASK_QUEUE_TIMEOUT = 0.01

logger = logging.getLogger('grab.spider.base')
logger_verbose = logging.getLogger('grab.spider.base.verbose')
# If you need verbose logging just
# change logging level of that logger
logger_verbose.setLevel(logging.FATAL)

class Spider(SpiderPattern, SpiderStat):
    """
    Asynchronious scraping framework.
    """

    # You can define here some urls and initial tasks
    # with name "initial" will be created from these
    # urls
    # If the logic of generating initial tasks is complex
    # then consider to use `task_generator` method instead of
    # `initial_urls` attribute
    initial_urls = None

    # The base url which is used to resolve all relative urls
    # The resolving takes place in `add_task` method
    base_url = None

    def __init__(self, thread_number=3,
                 network_try_limit=10, task_try_limit=10,
                 debug_error=False,
                 log_taskname=False,
                 request_pause=0,
                 priority_mode='random',
                 meta=None,
                 retry_rebuild_user_agent=True,
                 only_cache=False,
                 config=None,
                 slave=False,
                 ):
        """
        Arguments:
        * thread-number - Number of concurrent network streams
        * network_try_limit - How many times try to send request
            again if network error was occuried, use 0 to disable
        * network_try_limit - Limit of tries to execute some task
            this is not the same as network_try_limit
            network try limit limits the number of tries which
            are performed automaticall in case of network timeout
            of some other physical error
            but task_try_limit limits the number of attempts which
            are scheduled manually in the spider business logic
        * request_pause - amount of time on which the main `run` cycle should
            pause the activity of spider. By default it is equal to zero. You
            can use this option to slow down the spider speed (also you can use
            `thread_number` option). The value of `request_pause` could be float.
        * priority_mode - could be "random" or "const"
        * meta - arbitrary user data
        * retry_rebuid_user_agent - generate new random user-agent for each
            network request which is performed again due to network error
        """

        self.slave = slave

        self.timers = {}
        self.time_points = {}
        self.start_timer('total')
        if config is not None:
            self.config = config
        else:
            self.config = {}

        self.taskq = None

        if meta:
            self.meta = meta
        else:
            self.meta = {}

        self.task_generator_enabled = False
        self.only_cache = only_cache
        self.thread_number = thread_number
        self.counters = defaultdict(int)
        self.grab_config = {}
        self.items = {}
        self.task_try_limit = task_try_limit
        self.network_try_limit = network_try_limit
        if priority_mode not in ['random', 'const']:
            raise SpiderMisuseError('Value of priority_mode option should be "random" or "const"')
        else:
            self.priority_mode = priority_mode
        try:
            signal.signal(signal.SIGUSR1, self.sigusr1_handler)
        except (ValueError, AttributeError):
            pass
        try:
            signal.signal(signal.SIGUSR2, self.sigusr2_handler)
        except (ValueError, AttributeError):
            pass
        self.debug_error = debug_error

        # Initial cache-subsystem values
        self.cache_enabled = False
        self.cache = None

        self.log_taskname = log_taskname
        self.work_allowed = True
        self.request_pause = request_pause

        self.proxylist_enabled = None
        self.proxylist = None
        self.proxy = None
        self.proxy_auto_change = False
        self.retry_rebuild_user_agent = retry_rebuild_user_agent

    def setup_cache(self, backend='mongo', database=None, use_compression=True, **kwargs):
        if database is None:
            raise SpiderMisuseError('setup_cache method requires database option')
        self.cache_enabled = True
        mod = __import__('grab.spider.cache_backend.%s' % backend,
                         globals(), locals(), ['foo'])<commit_msg>Drop request_pause option<commit_after>"""
Global TODO:
* make task_%s_preprocess methods
"""
from __future__ import absolute_import
import types
import signal
import inspect
import traceback
import logging
from collections import defaultdict
import os
import time
import json
import cPickle as pickle
import anydbm
import multiprocessing
import zlib
from hashlib import sha1
from urlparse import urljoin
from random import randint

import Queue
from ..base import GLOBAL_STATE, Grab
from .error import SpiderError, SpiderMisuseError, FatalError
from .task import Task
from .data import Data
from .pattern import SpiderPattern
from .stat  import SpiderStat
from .transport.multicurl import MulticurlTransport
from ..proxylist import ProxyList

DEFAULT_TASK_PRIORITY = 100
RANDOM_TASK_PRIORITY_RANGE = (50, 100)
TASK_QUEUE_TIMEOUT = 0.01
NULL = object()

logger = logging.getLogger('grab.spider.base')
logger_verbose = logging.getLogger('grab.spider.base.verbose')
# If you need verbose logging just
# change logging level of that logger
logger_verbose.setLevel(logging.FATAL)

class Spider(SpiderPattern, SpiderStat):
    """
    Asynchronious scraping framework.
    """

    # You can define here some urls and initial tasks
    # with name "initial" will be created from these
    # urls
    # If the logic of generating initial tasks is complex
    # then consider to use `task_generator` method instead of
    # `initial_urls` attribute
    initial_urls = None

    # The base url which is used to resolve all relative urls
    # The resolving takes place in `add_task` method
    base_url = None

    def __init__(self, thread_number=3,
                 network_try_limit=10, task_try_limit=10,
                 debug_error=False,
                 log_taskname=False,
                 request_pause=NULL,
                 priority_mode='random',
                 meta=None,
                 retry_rebuild_user_agent=True,
                 only_cache=False,
                 config=None,
                 slave=False,
                 ):
        """
        Arguments:
        * thread-number - Number of concurrent network streams
        * network_try_limit - How many times try to send request
            again if network error was occuried, use 0 to disable
        * network_try_limit - Limit of tries to execute some task
            this is not the same as network_try_limit
            network try limit limits the number of tries which
            are performed automaticall in case of network timeout
            of some other physical error
            but task_try_limit limits the number of attempts which
            are scheduled manually in the spider business logic
        * priority_mode - could be "random" or "const"
        * meta - arbitrary user data
        * retry_rebuid_user_agent - generate new random user-agent for each
            network request which is performed again due to network error
        """

        self.slave = slave

        self.timers = {}
        self.time_points = {}
        self.start_timer('total')
        if config is not None:
            self.config = config
        else:
            self.config = {}

        self.taskq = None

        if meta:
            self.meta = meta
        else:
            self.meta = {}

        self.task_generator_enabled = False
        self.only_cache = only_cache
        self.thread_number = thread_number
        self.counters = defaultdict(int)
        self.grab_config = {}
        self.items = {}
        self.task_try_limit = task_try_limit
        self.network_try_limit = network_try_limit
        if priority_mode not in ['random', 'const']:
            raise SpiderMisuseError('Value of priority_mode option should be "random" or "const"')
        else:
            self.priority_mode = priority_mode
        try:
            signal.signal(signal.SIGUSR1, self.sigusr1_handler)
        except (ValueError, AttributeError):
            pass
        try:
            signal.signal(signal.SIGUSR2, self.sigusr2_handler)
        except (ValueError, AttributeError):
            pass
        self.debug_error = debug_error

        # Initial cache-subsystem values
        self.cache_enabled = False
        self.cache = None

        self.log_taskname = log_taskname
        self.work_allowed = True
        if request_pause is not NULL:
            logging.error('Option `request_pause` is deprecated and is not supported anymore')

        self.proxylist_enabled = None
        self.proxylist = None
        self.proxy = None
        self.proxy_auto_change = False
        self.retry_rebuild_user_agent = retry_rebuild_user_agent

    def setup_cache(self, backend='mongo', database=None, use_compression=True, **kwargs):
        if database is None:
            raise SpiderMisuseError('setup_cache method requires database option')
        self.cache_enabled = True
        mod = __import__('grab.spider.cache_backend.%s' % backend,
                         globals(), locals(), ['foo'])
{'fraction_non_alphanumeric': 0.06170901599136663, 'fraction_numerical': 0.0044147944667909346, 'mean_word_length': 2.3654671508748764, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4359038', 'n_tokens_mistral': 2724, 'n_tokens_neox': 2666, 'n_words': 892}