<commit_before>import importlib

from lamnfyc.logger import log  # noqa


def import_package(name, version):
    package_import = importlib.import_module(__package__ + '.packages.' + name)
    return package_import.VERSIONS[version]


def import_function(function_name):
    module, _, func = (function_name or '').partition(':')
    if module:
        try:
            return getattr(__import__(module), func)
        except AttributeError:
            raise ImportError('Could not import {}'.format(function_name))
    return None


def required_parameter(obj, name, message=None):
    message = message or 'Parameter {name} is required'
    if name not in obj:
        raise AttributeError(message.format(name=name))
    else:
        return obj[name]


class AttributeDict(dict):
    __getattr__ = dict.__getitem__
    __setattr__ = dict.__setitem__


def download(url, path, chunk_size=16 * 1024):
    with contextlib.closing(urllib2.urlopen(url)) as response, open(path, 'wb') as fileout:
        # endless loop to read `chunk_size` bytes in at a time until its done
        while True:<commit_msg>Adds better error messages when a package cant be found<commit_after>import importlib
import contextlib
import urllib2

from lamnfyc.logger import log  # noqa


def import_package(name, version):
    package_import = importlib.import_module(__package__ + '.packages.' + name)
    try:
        return package_import.VERSIONS[version]
    except KeyError:
        raise KeyError('{} {} was not found, these are the options: {}'.format(
            name, version, ', '.join(package_import.VERSIONS)
        ))


def import_function(function_name):
    module, _, func = (function_name or '').partition(':')
    if module:
        try:
            return getattr(__import__(module), func)
        except AttributeError:
            raise ImportError('Could not import {}'.format(function_name))
    return None


def required_parameter(obj, name, message=None):
    message = message or 'Parameter {name} is required'
    if name not in obj:
        raise AttributeError(message.format(name=name))
    else:
        return obj[name]


class AttributeDict(dict):
    __getattr__ = dict.__getitem__
    __setattr__ = dict.__setitem__


def download(url, path, chunk_size=16 * 1024):
    with contextlib.closing(urllib2.urlopen(url)) as response, open(path, 'wb') as fileout:
        # endless loop to read `chunk_size` bytes in at a time until its done
        while True:
{'fraction_non_alphanumeric': 0.09716599190283401, 'fraction_numerical': 0.006072874493927126, 'mean_word_length': 3.3967971530249113, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2079823', 'n_tokens_mistral': 726, 'n_tokens_neox': 703, 'n_words': 214}