<commit_before>import os
import re
import xbmcvfs
from contextlib import closing
from requests.exceptions import HTTPError, Timeout, ConnectionError, RequestException

from lib.libs import mediatypes, utils
from lib.libs.addonsettings import settings
from lib.libs.pykodi import localize as L
from lib.libs.webhelper import Getter

CANT_CONTACT_PROVIDER = 32034
HTTP_ERROR = 32035
CANT_WRITE_TO_FILE = 32037

typemap = {'image/jpeg': 'jpg', 'image/png': 'png', 'image/gif': 'gif'}

class Downloader(object):
    def __init__(self):
        self.getter = Getter()
        self.getter.session.headers['User-Agent'] = settings.useragent
        self.size = 0

    def downloadfor(self, mediaitem, allartwork=True):
        basefile = utils.find_central_infodir(mediaitem, True)
        path = basefile
        if not basefile:
            if not mediaitem.file or mediaitem.file.startswith(('plugin', 'http')):
                return False, '' # REVIEW: there may be other protocols that just can't be written to
            path = utils.get_movie_path_list(mediaitem.file)[0] \
                if mediaitem.mediatype == mediatypes.MOVIE else mediaitem.file
            basefile = os.path.splitext(path)[0]
        if allartwork:
            nowart = dict(mediaitem.art)
            nowart.update(mediaitem.selectedart)
        else:
            nowart = dict(mediaitem.selectedart)
        services_hit = False
        error = None
        def st(num):
            return '-specials' if num == 0 else '-all' if num == -1 else '{0:02d}'.format(num)
        seasonpre = None if mediaitem.mediatype != mediatypes.SEASON else 'season{0}-'.format(st(mediaitem.season))
        for arttype, url in nowart.iteritems():
            if not url or not url.startswith('http'):
                continue
            if seasonpre:
                type_for_file = seasonpre + arttype
            elif arttype.startswith('season.'):
                _, num, sarttype = arttype.split('.')
                type_for_file = 'season{0}-{1}'.format(st(int(num)), sarttype)
            else:
                type_for_file = arttype
            result, err = self.doget(url)
            if err:
                error = err
                continue
            if not result:
                # 404 URL dead, wipe it so we can add another one later
                mediaitem.downloadedart[arttype] = None
                continue
            self.size += int(result.headers.get('content-length', 0))
            services_hit = True
            contenttype = result.headers.get('content-type')
            if contenttype in typemap:
                ext = typemap[contenttype]
            else:
                if not re.search('\.\w*$', url):
                    continue
                ext = url.rsplit('.', 1)[1]

            filename = basefile + ('-' if basefile != path else '') + type_for_file + '.' + ext
            # For now this just downloads the whole thing in memory, then saves it to file.
            #  Maybe chunking it will be better when GIFs are handled
            with closing(xbmcvfs.File(filename, 'wb')) as file_:
                if not file_.write(result.content):
                    raise DownloaderError(L(CANT_WRITE_TO_FILE).format(filename))
            mediaitem.downloadedart[arttype] = filename
        return services_hit, error

    def doget(self, url, **kwargs):
        try:
            return self.getter(url, **kwargs), None
        except (Timeout, ConnectionError) as ex:
            return None, L(CANT_CONTACT_PROVIDER)
        except HTTPError as ex:
            message = ex.response.reason if ex.response else type(ex).__name__
            return None, L(HTTP_ERROR).format(message)
        except RequestException as ex:
            return None, L(HTTP_ERROR).format(type(ex).__name__)

class DownloaderError(Exception):
    def __init__(self, message, cause=None):
        super(DownloaderError, self).__init__(message)
        self.cause = cause<commit_msg>fix: don't create a movie set central dir unless there is something to save<commit_after>import os
import re
import xbmcvfs
from contextlib import closing
from requests.exceptions import HTTPError, Timeout, ConnectionError, RequestException

from lib.libs import mediatypes, utils
from lib.libs.addonsettings import settings
from lib.libs.pykodi import localize as L
from lib.libs.webhelper import Getter

CANT_CONTACT_PROVIDER = 32034
HTTP_ERROR = 32035
CANT_WRITE_TO_FILE = 32037

typemap = {'image/jpeg': 'jpg', 'image/png': 'png', 'image/gif': 'gif'}

class Downloader(object):
    def __init__(self):
        self.getter = Getter()
        self.getter.session.headers['User-Agent'] = settings.useragent
        self.size = 0

    def downloadfor(self, mediaitem, allartwork=True):
        if allartwork:
            nowart = dict(mediaitem.art)
            nowart.update(mediaitem.selectedart)
        else:
            nowart = dict(mediaitem.selectedart)
        if not something_todownload(nowart):
            return False, ''
        basefile = utils.find_central_infodir(mediaitem, True)
        path = basefile
        if not basefile:
            if not mediaitem.file or mediaitem.file.startswith(('plugin', 'http')):
                return False, '' # REVIEW: there may be other protocols that just can't be written to
            path = utils.get_movie_path_list(mediaitem.file)[0] \
                if mediaitem.mediatype == mediatypes.MOVIE else mediaitem.file
            basefile = os.path.splitext(path)[0]
        services_hit = False
        error = None
        def st(num):
            return '-specials' if num == 0 else '-all' if num == -1 else '{0:02d}'.format(num)
        seasonpre = None if mediaitem.mediatype != mediatypes.SEASON else 'season{0}-'.format(st(mediaitem.season))
        for arttype, url in nowart.iteritems():
            if not url or not url.startswith('http'):
                continue
            if seasonpre:
                type_for_file = seasonpre + arttype
            elif arttype.startswith('season.'):
                _, num, sarttype = arttype.split('.')
                type_for_file = 'season{0}-{1}'.format(st(int(num)), sarttype)
            else:
                type_for_file = arttype
            result, err = self.doget(url)
            if err:
                error = err
                continue
            if not result:
                # 404 URL dead, wipe it so we can add another one later
                mediaitem.downloadedart[arttype] = None
                continue
            self.size += int(result.headers.get('content-length', 0))
            services_hit = True
            contenttype = result.headers.get('content-type')
            if contenttype in typemap:
                ext = typemap[contenttype]
            else:
                if not re.search('\.\w*$', url):
                    continue
                ext = url.rsplit('.', 1)[1]

            filename = basefile + ('-' if basefile != path else '') + type_for_file + '.' + ext
            # For now this just downloads the whole thing in memory, then saves it to file.
            #  Maybe chunking it will be better when GIFs are handled
            with closing(xbmcvfs.File(filename, 'wb')) as file_:
                if not file_.write(result.content):
                    raise DownloaderError(L(CANT_WRITE_TO_FILE).format(filename))
            mediaitem.downloadedart[arttype] = filename
        return services_hit, error

    def doget(self, url, **kwargs):
        try:
            return self.getter(url, **kwargs), None
        except (Timeout, ConnectionError) as ex:
            return None, L(CANT_CONTACT_PROVIDER)
        except HTTPError as ex:
            message = ex.response.reason if ex.response else type(ex).__name__
            return None, L(HTTP_ERROR).format(message)
        except RequestException as ex:
            return None, L(HTTP_ERROR).format(type(ex).__name__)

def something_todownload(artmap):
    for arttype, url in artmap.iteritems():
        if url and url.startswith('http'):
            return True
    return False

class DownloaderError(Exception):
    def __init__(self, message, cause=None):
        super(DownloaderError, self).__init__(message)
        self.cause = cause
{'fraction_non_alphanumeric': 0.09712970812643817, 'fraction_numerical': 0.007750999152234468, 'mean_word_length': 2.1495041952707856, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5477691', 'n_tokens_mistral': 2416, 'n_tokens_neox': 2315, 'n_words': 640}