<commit_before>from __future__ import absolute_import
import os
import logging
import time
import kaa

from ..utils import Curl, download
from ..curl import CurlError
from ..config import config
from .base import RetrieverBase, RetrieverError, RetrieverAbortedSoft

__all__ = ['Retriever']

log = logging.getLogger('stagehand.retrievers.http')

class Retriever(RetrieverBase):
    NAME = 'http'
    PRINTABLE_NAME = 'HTTP'
    SUPPORTED_TYPES = ('http',)
    ALWAYS_ENABLED = True

    @kaa.timed(1)
    def _verify_timer(self, progress, episode, result, outfile, curl_ip):
        if progress.pos < 512:
            # Wait until we have 512KB before checking file.
            return
        try:
            r = self.verify_result_file(episode, result, outfile)
        except RetrieverError as e:
            # Verify failed, abort download.
            curl_ip.abort(RetrieverAbortedSoft(*e.args))
            return False
        else:
            if r is not False:
                # verify function returned either True (verified ok) or None
                # (no ability to get metadata).  Either way, stop the timer.
                return False


    def _download_progress_cb(self, curl, state, position, total, speed, progress):<commit_msg>Silence harmless unhandled asynchronous exception during abort<commit_after>from __future__ import absolute_import
import os
import logging
import time
import kaa

from ..utils import Curl, download
from ..curl import CurlError
from ..config import config
from .base import RetrieverBase, RetrieverError, RetrieverAbortedSoft

__all__ = ['Retriever']

log = logging.getLogger('stagehand.retrievers.http')

class Retriever(RetrieverBase):
    NAME = 'http'
    PRINTABLE_NAME = 'HTTP'
    SUPPORTED_TYPES = ('http',)
    ALWAYS_ENABLED = True

    @kaa.timed(1)
    def _verify_timer(self, progress, episode, result, outfile, curl_ip):
        if progress.pos < 512:
            # Wait until we have 512KB before checking file.
            return
        try:
            r = self.verify_result_file(episode, result, outfile)
        except RetrieverError as e:
            # Verify failed, abort download.
            try:
                curl_ip.abort(RetrieverAbortedSoft(*e.args))
            except kaa.InProgressAborted:
                # curl doesn't handle this so abort() reraises.  We can just
                # absorb this silently, since it will bubble up through
                # _retrieve()
                pass
            return False
        else:
            if r is not False:
                # verify function returned either True (verified ok) or None
                # (no ability to get metadata).  Either way, stop the timer.
                return False


    def _download_progress_cb(self, curl, state, position, total, speed, progress):
{'fraction_non_alphanumeric': 0.07350852272727272, 'fraction_numerical': 0.004971590909090909, 'mean_word_length': 2.431181485992692, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2189512', 'n_tokens_mistral': 816, 'n_tokens_neox': 776, 'n_words': 261}