<commit_before>import os
from time import mktime
from dateutil.parser import parse
from binstar_client import errors


class Downloader(object):
    """
    Download notebook from anaconda.org
    """
    def __init__(self, binstar, username, notebook):
        self.binstar = binstar
        self.username = username
        self.notebook = notebook

    def __call__(self, output='.', force=False):
        self.output = output
        self.ensure_output(force)
        return self.download_files(force)

    def download_files(self, force=False):
        output = []
        for f in self.list_files():
            if self.can_download(f, force):
                self.download(f)
                output.append(f['basename'])
        return output

    def download(self, dist):
        """
        Download file into location
        """
        requests_handle = self.binstar.download(self.username, self.notebook,
                                                dist['version'], dist['basename'])

        with open(os.path.join(self.output, dist['basename']), 'wb') as fdout:
            for chunk in requests_handle.iter_content(4096):
                fdout.write(chunk)

    def can_download(self, dist, force=False):
        """
        Can download if location/file does not exist or if force=True
        :param dist:
        :param force:
        :return: True/False
        """
        return not os.path.exists(os.path.join(self.output, dist['basename'])) or force

    def ensure_output(self, force):
        """
        Ensure output's directory exists
        """
        if self.output == '.':
            return
        if not os.path.exists(self.output):
            os.makedirs(self.output)
        elif not force:
            raise errors.DestionationPathExists(self.output)

    def list_files(self):
        """
        List available files in a project (aka notebook)
        :return: list
        """
        output = []
        tmp = {}

        files = self.binstar.package(self.username, self.notebook)['files']

        for f in files:
            if f['basename'] in tmp:
                tmp[f['basename']].append(f)
            else:
                tmp[f['basename']] = [f]

        for basename, versions in tmp.items():
            try:
                output.append(max(versions, key=lambda x: int(x['version'])))
            except ValueError:
                output.append(
                    max(versions, key=lambda x: mktime(parse(x['upload_time']).timetuple()))
                )
            except:
                output.append(versions[-1])

        return output
<commit_msg>If folder does not exists, create it<commit_after>import os
from time import mktime
from dateutil.parser import parse
from binstar_client import errors


class Downloader(object):
    """
    Download notebook from anaconda.org
    """
    def __init__(self, binstar, username, notebook):
        self.binstar = binstar
        self.username = username
        self.notebook = notebook

    def __call__(self, output='.', force=False):
        self.output = output
        self.ensure_output()
        return self.download_files(force)

    def download_files(self, force=False):
        output = []
        for f in self.list_files():
            if self.can_download(f, force):
                self.download(f)
                output.append(f['basename'])
        return output

    def download(self, dist):
        """
        Download file into location
        """
        requests_handle = self.binstar.download(self.username, self.notebook,
                                                dist['version'], dist['basename'])

        with open(os.path.join(self.output, dist['basename']), 'wb') as fdout:
            for chunk in requests_handle.iter_content(4096):
                fdout.write(chunk)

    def can_download(self, dist, force=False):
        """
        Can download if location/file does not exist or if force=True
        :param dist:
        :param force:
        :return: True/False
        """
        return not os.path.exists(os.path.join(self.output, dist['basename'])) or force

    def ensure_output(self):
        """
        Ensure output's directory exists
        """
        if not os.path.exists(self.output):
            os.makedirs(self.output)

    def list_files(self):
        """
        List available files in a project (aka notebook)
        :return: list
        """
        output = []
        tmp = {}

        files = self.binstar.package(self.username, self.notebook)['files']

        for f in files:
            if f['basename'] in tmp:
                tmp[f['basename']].append(f)
            else:
                tmp[f['basename']] = [f]

        for basename, versions in tmp.items():
            try:
                output.append(max(versions, key=lambda x: int(x['version'])))
            except ValueError:
                output.append(
                    max(versions, key=lambda x: mktime(parse(x['upload_time']).timetuple()))
                )
            except:
                output.append(versions[-1])

        return output

{'fraction_non_alphanumeric': 0.112109375, 'fraction_numerical': 0.001953125, 'mean_word_length': 1.829281767955801, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4430167', 'n_tokens_mistral': 1414, 'n_tokens_neox': 1363, 'n_words': 343}