<commit_before>import re

import collections
import logbook
import requests

from nameko.events import event_handler

from .logger import LoggingDependency
from .storages import RedisStorage

HEAD_TIMEOUT = 10  # in seconds
GET_TIMEOUT = 3 * 60  # in seconds

# See https://github.com/kvesteri/validators for reference.
url_pattern = re.compile(
    r'^[a-z]+://([^/:]+\.[a-z]{2,10}|([0-9]{{1,3}}\.)'
    r'{{3}}[0-9]{{1,3}})(:[0-9]+)?(\/.*)?$'
)

log = logbook.debug
FakeResponse = collections.namedtuple('Response', ['status_code', 'headers'])
session = requests.Session()
adapter = requests.adapters.HTTPAdapter(pool_connections=20, pool_maxsize=50)
session.mount('http://', adapter)


class CrawlerService(object):
    name = 'url_crawler'
    storage = RedisStorage()
    logger = LoggingDependency()

    @event_handler('http_server', 'url_to_check')
    @event_handler('timer', 'url_to_check')
    def check_url(self, url_group_frequency):
        url, group, frequency = url_group_frequency
        log(('Checking {url} for group {group} and frequency "{frequency}"'
             .format(url=url, group=group, frequency=frequency)))
        if not url_pattern.match(url):
            logbook.error('Error with {url}: not a URL'.format(url=url))
            return
        self.storage.store_url(url)
        if group:
            self.storage.store_group(url, group)
            if frequency:
                self.storage.store_frequency(url, group, frequency)
        try:
            try:
                response = session.head(url, allow_redirects=True,
                                        timeout=HEAD_TIMEOUT)
            except requests.exceptions.ReadTimeout:
                # simulate 404 to trigger GET request below
                response = FakeResponse(status_code=404, headers={})
            # Double check for servers not dealing properly with HEAD.
            if response.status_code in (400, 404, 405):
                log('Checking {url} with a GET'.format(url=url))
                response = session.get(url, allow_redirects=True,
                                       timeout=GET_TIMEOUT)
        except (requests.exceptions.ConnectionError,
                requests.exceptions.ReadTimeout):
            response = FakeResponse(status_code=503, headers={})
        except Exception as e:
            logbook.error('Error with {url}: {e}'.format(url=url, e=e))
            return
        self.storage.store_metadata(url, response)<commit_msg>Use stream when GET to avoid body download<commit_after>import re

import collections
import logbook
import requests

from nameko.events import event_handler

from .logger import LoggingDependency
from .storages import RedisStorage

HEAD_TIMEOUT = 10  # in seconds
GET_TIMEOUT = 3 * 60  # in seconds

# See https://github.com/kvesteri/validators for reference.
url_pattern = re.compile(
    r'^[a-z]+://([^/:]+\.[a-z]{2,10}|([0-9]{{1,3}}\.)'
    r'{{3}}[0-9]{{1,3}})(:[0-9]+)?(\/.*)?$'
)

log = logbook.debug
FakeResponse = collections.namedtuple('Response', ['status_code', 'headers'])
session = requests.Session()
adapter = requests.adapters.HTTPAdapter(pool_connections=20, pool_maxsize=50)
session.mount('http://', adapter)


class CrawlerService(object):
    name = 'url_crawler'
    storage = RedisStorage()
    logger = LoggingDependency()

    @event_handler('http_server', 'url_to_check')
    @event_handler('timer', 'url_to_check')
    def check_url(self, url_group_frequency):
        url, group, frequency = url_group_frequency
        log(('Checking {url} for group {group} and frequency "{frequency}"'
             .format(url=url, group=group, frequency=frequency)))
        if not url_pattern.match(url):
            logbook.error('Error with {url}: not a URL'.format(url=url))
            return
        self.storage.store_url(url)
        if group:
            self.storage.store_group(url, group)
            if frequency:
                self.storage.store_frequency(url, group, frequency)
        try:
            try:
                response = session.head(url, allow_redirects=True,
                                        timeout=HEAD_TIMEOUT)
            except requests.exceptions.ReadTimeout:
                # simulate 404 to trigger GET request below
                response = FakeResponse(status_code=404, headers={})
            # Double check for servers not dealing properly with HEAD.
            if response.status_code in (400, 404, 405):
                log('Checking {url} with a GET'.format(url=url))
                response = session.get(url, allow_redirects=True,
                                       timeout=GET_TIMEOUT, stream=True)
        except (requests.exceptions.ConnectionError,
                requests.exceptions.ReadTimeout):
            response = FakeResponse(status_code=503, headers={})
        except Exception as e:
            logbook.error('Error with {url}: {e}'.format(url=url, e=e))
            return
        self.storage.store_metadata(url, response)
{'fraction_non_alphanumeric': 0.1224, 'fraction_numerical': 0.0164, 'mean_word_length': 2.600431965442765, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5356314', 'n_tokens_mistral': 1484, 'n_tokens_neox': 1408, 'n_words': 327}