<commit_before>"""
homeassistant.components.downloader
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Provides functionality to download files.
"""
import os
import logging
import re

import homeassistant.util as util

DOMAIN = "downloader"

SERVICE_DOWNLOAD_FILE = "download_file"


def setup(bus, download_path):
    """ Listens for download events to download files. """

    logger = logging.getLogger(__name__)

    try:
        import requests
    except ImportError:
        logger.exception(("Failed to import requests. "
                          "Did you maybe not execute 'pip install requests'?"))

        return False

    if not os.path.isdir(download_path):

        logger.error(
            ("Download path {} does not exist. File Downloader not active.").
            format(download_path))

        return False

    def _download_file(service):
        """ Downloads file specified in the url. """

        try:
            req = requests.get(service.data['url'], stream=True)
            if req.status_code == 200:
                filename = None

                if 'content-disposition' in req.headers:
                    match = re.findall(r"filename=(\S+)",
                                       req.headers['content-disposition'])

                    if len(match) > 0:
                        filename = match[0].strip("'\" ")

                if not filename:
                    filename = os.path.basename(service.data['url']).strip()

                if not filename:
                    filename = "ha_download"

                # Remove stuff to ruin paths
                filename = util.sanitize_filename(filename)

                path, ext = os.path.splitext(os.path.join(download_path,
                                                          filename))

                # If file exist append a number. We test filename, filename_2..
                tries = 1
                final_path = path + ext
                while os.path.isfile(final_path):
                    tries += 1

                    final_path = path + "_{}".format(tries) + ext

                logger.info("{} -> {}".format(
                            service.data['url'], final_path))

                with open(final_path, 'wb') as fil:
                    for chunk in req.iter_content(1024):
                        fil.write(chunk)

        except requests.exceptions.ConnectionError:
            logger.exception("ConnectionError occured for {}".
                             format(service.data['url']))

    bus.register_service(DOMAIN, SERVICE_DOWNLOAD_FILE,
                         _download_file)

    return True<commit_msg>Downloader downloads file in seperate thread now so it doesn't block bus queue<commit_after>"""
homeassistant.components.downloader
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Provides functionality to download files.
"""
import os
import logging
import re
import threading

import homeassistant.util as util

DOMAIN = "downloader"

SERVICE_DOWNLOAD_FILE = "download_file"


# pylint: disable=too-many-branches
def setup(bus, download_path):
    """ Listens for download events to download files. """

    logger = logging.getLogger(__name__)

    try:
        import requests
    except ImportError:
        logger.exception(("Failed to import requests. "
                          "Did you maybe not execute 'pip install requests'?"))

        return False

    if not os.path.isdir(download_path):

        logger.error(
            ("Download path {} does not exist. File Downloader not active.").
            format(download_path))

        return False

    def download_file(service):
        """ Starts thread to download file specified in the url. """

        if not 'url' in service.data:
            logger.error("Service called but 'url' parameter not specified.")
            return

        def do_download():
            """ Downloads the file. """
            try:
                url = service.data['url']
                req = requests.get(url, stream=True)

                if req.status_code == 200:
                    filename = None

                    if 'content-disposition' in req.headers:
                        match = re.findall(r"filename=(\S+)",
                                           req.headers['content-disposition'])

                        if len(match) > 0:
                            filename = match[0].strip("'\" ")

                    if not filename:
                        filename = os.path.basename(
                            url).strip()

                    if not filename:
                        filename = "ha_download"

                    # Remove stuff to ruin paths
                    filename = util.sanitize_filename(filename)

                    path, ext = os.path.splitext(os.path.join(download_path,
                                                              filename))

                    # If file exist append a number.
                    # We test filename, filename_2..
                    tries = 1
                    final_path = path + ext
                    while os.path.isfile(final_path):
                        tries += 1

                        final_path = path + "_{}".format(tries) + ext

                    logger.info("{} -> {}".format(
                                url, final_path))

                    with open(final_path, 'wb') as fil:
                        for chunk in req.iter_content(1024):
                            fil.write(chunk)

                    logger.info("Downloading of {} done".format(
                        url))

            except requests.exceptions.ConnectionError:
                logger.exception("ConnectionError occured for {}".
                                 format(url))

        threading.Thread(target=do_download).start()

    bus.register_service(DOMAIN, SERVICE_DOWNLOAD_FILE,
                         download_file)

    return True
{'fraction_non_alphanumeric': 0.1009282700421941, 'fraction_numerical': 0.004050632911392405, 'mean_word_length': 1.4437113402061856, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4808512', 'n_tokens_mistral': 1480, 'n_tokens_neox': 1350, 'n_words': 376}