<commit_before>from __future__ import with_statement

import hashlib
import logging
import tempfile
import urllib2

from django.conf import settings

from django_longliving.base import LonglivingThread

logger = logging.getLogger(__name__)

class Downloader(LonglivingThread):
    QUEUE_NAME = 'downloader:queue'

    DEFAULT_USER_AGENT = 'Mozilla (compatible; Humfrey downloader; https://github.com/oucs/humfrey)'
    DEFAULT_ACCEPT = 'application/rdf+xml, text/n3, text/turtle, application/xhtml+xml;q=0.9, text/html;q=0.9'

    SAVE_BASE = settings.DOWNLOADER_DEFAULT_DIR or None

    def run(self):
        client = self.get_redis_client()

        for _, item in self.watch_queue(client, 'downloader:queue', True):
            try:
                self.download_item(client, item)
            except Exception:
                logger.exception('Failed to download item')
                failure_queue = item.get('failure_queue', 'downloader:failures')
                client.rpush(failure_queue, self.pack(item))

    def download_item(self, client, item):
        url_hash = hashlib.sha1(item['url']).hexdigest()

        headers = {
            'User-Agent': self.DEFAULT_USER_AGENT,
            'Accept': self.DEFAULT_ACCEPT,
        }
        if 'headers' in item:
            headers.update(item['headers'])
        if 'accept' in item:
            headers['Accept'] = item['accept']

        request = urllib2.Request(item['url'])
        for key, value in headers.iteritems():
            request.headers[key] = value

        target_queue = item['target_queue']
        failure_queue = item.get('failure_queue', 'downloader:failures')

        if item.get('conditional_caching'):
            etag = client.hget('downloader:etag', url_hash)
            last_modified = client.hget('downloader:last-modified', url_hash)

            if etag:
                headers['If-None-Match'] = etag
            if last_modified:
                headers['If-Modified-Since'] = last_modified

        response = None
        try:
            response = urllib2.urlopen(request)
        except urllib2.URLError, e:
            item['error'] = repr(e)
            logging.exception("Couldn't retrieve file")
        except urllib2.HTTPError, e:
            if e.code in (204, 304):
                response = e
            else:
                item['status_code'] = e.code
                item['error'] = repr(e)<commit_msg>Downloader now places items in the target queue if a failure queue isn't specified explicitly.<commit_after>from __future__ import with_statement

import hashlib
import logging
import tempfile
import urllib2

from django.conf import settings

from django_longliving.base import LonglivingThread

logger = logging.getLogger(__name__)

class Downloader(LonglivingThread):
    QUEUE_NAME = 'downloader:queue'

    DEFAULT_USER_AGENT = 'Mozilla (compatible; Humfrey downloader; https://github.com/oucs/humfrey)'
    DEFAULT_ACCEPT = 'application/rdf+xml, text/n3, text/turtle, application/xhtml+xml;q=0.9, text/html;q=0.9'

    SAVE_BASE = settings.DOWNLOADER_DEFAULT_DIR or None

    def run(self):
        client = self.get_redis_client()

        for _, item in self.watch_queue(client, 'downloader:queue', True):
            try:
                self.download_item(client, item)
            except Exception:
                logger.exception('Failed to download item')
                failure_queue = item.get('failure_queue', 'downloader:failures')
                client.rpush(failure_queue, self.pack(item))

    def download_item(self, client, item):
        url_hash = hashlib.sha1(item['url']).hexdigest()

        headers = {
            'User-Agent': self.DEFAULT_USER_AGENT,
            'Accept': self.DEFAULT_ACCEPT,
        }
        if 'headers' in item:
            headers.update(item['headers'])
        if 'accept' in item:
            headers['Accept'] = item['accept']

        request = urllib2.Request(item['url'])
        for key, value in headers.iteritems():
            request.headers[key] = value

        target_queue = item['target_queue']
        failure_queue = item.get('failure_queue', target_queue)

        if item.get('conditional_caching'):
            etag = client.hget('downloader:etag', url_hash)
            last_modified = client.hget('downloader:last-modified', url_hash)

            if etag:
                headers['If-None-Match'] = etag
            if last_modified:
                headers['If-Modified-Since'] = last_modified

        response = None
        try:
            response = urllib2.urlopen(request)
        except urllib2.URLError, e:
            item['error'] = repr(e)
            logging.exception("Couldn't retrieve file")
        except urllib2.HTTPError, e:
            if e.code in (204, 304):
                response = e
            else:
                item['status_code'] = e.code
                item['error'] = repr(e)
{'fraction_non_alphanumeric': 0.107157381049207, 'fraction_numerical': 0.0069133794225294835, 'mean_word_length': 2.3645690834473325, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '917722', 'n_tokens_mistral': 1469, 'n_tokens_neox': 1392, 'n_words': 305}