<commit_before>import errno
import os
import logging
import urllib2
import urlparse
import m3u8

import crypto

def consume_from_balancer(balancer, destination, encrypt=False):
    '''
    Consume all active playlist resources from ``balancer`` and
    report status to it.

    '''
    for playlist_resource in balancer.actives:
        try:
            m3u8_uri = str(playlist_resource)
            modified = consume(m3u8_uri, destination, encrypt)
        except (urllib2.HTTPError, IOError, OSError) as err:
            logging.warning(u'Notifying error for resource %s: %s' % (playlist_resource, err))
            balancer.notify_error(playlist_resource.server, playlist_resource.path)
        else:
            if modified:
                logging.info('Notifying content modified: %s' % playlist_resource)
                balancer.notify_modified(playlist_resource.server, playlist_resource.path)
            else:
                logging.debug('Content not modified: %s' % playlist_resource)


def consume(m3u8_uri, destination_path, encrypt=False):
    '''
    Given a ``m3u8_uri``, downloads all files to disk
    The remote path structure is maintained under ``destination_path``
<commit_msg>consumer: Add httplib.HTTPException catching<commit_after>import errno
import os
import logging
import urllib2
import httplib
import urlparse
import m3u8

import crypto

def consume_from_balancer(balancer, destination, encrypt=False):
    '''
    Consume all active playlist resources from ``balancer`` and
    report status to it.

    '''
    for playlist_resource in balancer.actives:
        try:
            m3u8_uri = str(playlist_resource)
            modified = consume(m3u8_uri, destination, encrypt)
        except (httplib.HTTPException, urllib2.HTTPError, IOError, OSError) as err:
            logging.warning(u'Notifying error for resource %s: %s' % (playlist_resource, err))
            balancer.notify_error(playlist_resource.server, playlist_resource.path)
        else:
            if modified:
                logging.info('Notifying content modified: %s' % playlist_resource)
                balancer.notify_modified(playlist_resource.server, playlist_resource.path)
            else:
                logging.debug('Content not modified: %s' % playlist_resource)


def consume(m3u8_uri, destination_path, encrypt=False):
    '''
    Given a ``m3u8_uri``, downloads all files to disk
    The remote path structure is maintained under ``destination_path``

{'fraction_non_alphanumeric': 0.08074283407347597, 'fraction_numerical': 0.009689140088817117, 'mean_word_length': 3.150753768844221, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3267477', 'n_tokens_mistral': 721, 'n_tokens_neox': 701, 'n_words': 192}