<commit_before># This file is part of fedmsg-downloader.
# Copyright (C) 2012 Red Hat, Inc.
#
# fedmsg-downloader is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# fedmsg-downloader is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with fedmsg; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#
# Authors:  <NAME> <<EMAIL>>
#
import fedmsg
import re
import os
from fedmsg.consumers import FedmsgConsumer
from fedmsg_download.download import Downloader
from threading import Thread
from Queue import Queue

import logging
log = logging.getLogger(__name__)

def download_thread(queue):
    while True:
        download = queue.get()
        download.sync_it_down()
        queue.task_done()

class RsyncConsumer(FedmsgConsumer):
    topic = "org.fedoraproject.*"
    config_key = 'fedmsg-download.consumer.enabled'
    rsync_queue = Queue()

    def __init__(self, hub):
        settings = hub.config.get('download')
        self.local_path = settings.get('local_path')
        self.rsync_base = settings.get('rsync_base')
        self.rsync_opts = settings.get('rsync_opts')
        self.delete_old = settings.get('delete_old')
        self.req_compose = settings.get('req_compose', True)
        self.ignore_name = settings.get('ignore_name', False)
        self.import_command = settings.get('import_command')
        self.regex_topic = re.compile(settings.get('filter_topic', ''))
        if not self.local_path:
            raise Exception('local_path not configured!')

        worker = Thread(target=download_thread, args=(self.rsync_queue,))
        worker.setDaemon(True)
        worker.start()

        super(RsyncConsumer, self).__init__(hub)
<commit_msg>wrap download method in try/except block.<commit_after># This file is part of fedmsg-downloader.
# Copyright (C) 2012 Red Hat, Inc.
#
# fedmsg-downloader is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# fedmsg-downloader is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with fedmsg; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#
# Authors:  <NAME> <<EMAIL>>
#
import fedmsg
import re
import os
from fedmsg.consumers import FedmsgConsumer
from fedmsg_download.download import Downloader
from threading import Thread
from Queue import Queue

import logging
log = logging.getLogger(__name__)

def download_thread(queue):
    while True:
        download = queue.get()
        try:
            download.sync_it_down()
        except Exception, e:
            log.critical(e)
        queue.task_done()

class RsyncConsumer(FedmsgConsumer):
    topic = "org.fedoraproject.*"
    config_key = 'fedmsg-download.consumer.enabled'
    rsync_queue = Queue()

    def __init__(self, hub):
        settings = hub.config.get('download')
        self.local_path = settings.get('local_path')
        self.rsync_base = settings.get('rsync_base')
        self.rsync_opts = settings.get('rsync_opts')
        self.delete_old = settings.get('delete_old')
        self.req_compose = settings.get('req_compose', True)
        self.ignore_name = settings.get('ignore_name', False)
        self.import_command = settings.get('import_command')
        self.regex_topic = re.compile(settings.get('filter_topic', ''))
        if not self.local_path:
            raise Exception('local_path not configured!')

        worker = Thread(target=download_thread, args=(self.rsync_queue,))
        worker.setDaemon(True)
        worker.start()

        super(RsyncConsumer, self).__init__(hub)

{'fraction_non_alphanumeric': 0.09082813891362422, 'fraction_numerical': 0.007569011576135352, 'mean_word_length': 3.603483606557377, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 25, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5029883', 'n_tokens_mistral': 1296, 'n_tokens_neox': 1256, 'n_words': 426}