<commit_before>#! /usr/bin/env python
import os
import urllib
import zipfile
import tempfile
import warnings
import shutil

import yaml

from ..babel import query_config_var
from ..utils.run_dir import cd


def load_default_parameters(path):
    with open(path, 'r') as fp:
        param_groups = yaml.load_all(fp.read())

    p = {}
    for group in param_groups:
        for name, param in group.items():
            if not name.startswith('_'):
                p[name] = param['value']['default']
    return p


def load_info(path):
    with open(path, 'r') as fp:
        info = yaml.load(fp.read())
    return info


def update_values(values, defaults):
    if not set(values.keys()).issubset(defaults.keys()):
        raise ValueError('Unknown values')

    updated = defaults.copy()
    updated.update(values)
    return updated


class SetupMixIn(object):
    def __init__(self):
        datarootdir = query_config_var('datarootdir')
        name = self.__class__.__name__.split('.')[-1]
        self._datadir = os.path.join(datarootdir, 'csdms', name)
        self._defaults = load_default_parameters(
            os.path.join(self.datadir, 'parameters.yaml')) 
        self._info = load_info(os.path.join(self.datadir, 'info.yaml'))

    def setup(self, dir=None, values=None):
        if values is None:
            values = dict(self.defaults)
        else:
            values = update_values(values, self._defaults)

        dir = dir or tempfile.mkdtemp()

        files = []
        for file_ in os.listdir(self.datadir):
            if file_ not in ('api.yaml', 'parameters.yaml', 'info.yaml'):
                files.append(os.path.join(self.datadir, file_))

        with cd(dir, create=True):
            for file_ in files:
                (base, ext) = os.path.splitext(file_)
                if ext == '.tmpl':
                    with open(file_, 'r') as fp:
                        template = fp.read()
                    (_, name) = os.path.split(base)
                    with open(name, 'w') as fp:
                        fp.write(template.format(**values))
                else:
                    shutil.copy(file_, '.')

        return dir

    @property
    def defaults(self):
        return self._defaults.items()

    @property
    def datadir(self):
        return self._datadir

    @property
    def author(self):
        try:
            return '{author} <{email}>'.format(**self._info)
        except KeyError:
            return self._info.get('author', 'unknown')

    @property
    def url(self):
        return self._info.get('url', 'unknown')

    @property
    def license(self):
        return self._info.get('license', 'unknown')

    @property
    def doi(self):<commit_msg>Read units from parameters.yaml file.<commit_after>#! /usr/bin/env python
import os
import urllib
import zipfile
import tempfile
import warnings
import shutil

import yaml

from ..babel import query_config_var
from ..utils.run_dir import cd


def load_default_parameters(path):
    with open(path, 'r') as fp:
        param_groups = yaml.load_all(fp.read())

    p = {}
    for group in param_groups:
        for name, param in group.items():
            if not name.startswith('_'):
                value = param['value']['default']
                try:
                    units = param['value']['units']
                except KeyError:
                    units = '-'
                    warnings.warn('missing units for {name}'.format(name=name))
                p[name] = (value, units)
    return p


def load_info(path):
    with open(path, 'r') as fp:
        info = yaml.load(fp.read())
    return info


def update_values(values, defaults):
    if not set(values.keys()).issubset(defaults.keys()):
        raise ValueError('Unknown values')

    updated = defaults.copy()
    updated.update(values)
    return updated


class SetupMixIn(object):
    def __init__(self):
        datarootdir = query_config_var('datarootdir')
        name = self.__class__.__name__.split('.')[-1]
        self._datadir = os.path.join(datarootdir, 'csdms', name)
        self._defaults = load_default_parameters(
            os.path.join(self.datadir, 'parameters.yaml')) 
        self._info = load_info(os.path.join(self.datadir, 'info.yaml'))
        self._parameters = {}
        for name, val_and_units in self._defaults.items():
            self._parameters[name] = val_and_units[0]

    def setup(self, *args, **kwds):
        if len(args) == 0:
            dir = tempfile.mkdtemp()
        else:
            dir = args[0]

        self._parameters.update(kwds)

        files = []
        for file_ in os.listdir(self.datadir):
            if file_ not in ('api.yaml', 'parameters.yaml', 'info.yaml'):
                files.append(os.path.join(self.datadir, file_))

        with cd(dir, create=True):
            for file_ in files:
                (base, ext) = os.path.splitext(file_)
                if ext == '.tmpl':
                    with open(file_, 'r') as fp:
                        template = fp.read()
                    (_, name) = os.path.split(base)
                    with open(name, 'w') as fp:
                        fp.write(template.format(**self._parameters))
                else:
                    shutil.copy(file_, '.')

        return dir

    @property
    def parameters(self):
        return self._parameters.items()

    @property
    def defaults(self):
        return self._defaults.items()

    @property
    def datadir(self):
        return self._datadir

    @property
    def author(self):
        try:
            return '{author} <{email}>'.format(**self._info)
        except KeyError:
            return self._info.get('author', 'unknown')

    @property
    def url(self):
        return self._info.get('url', 'unknown')

    @property
    def license(self):
        return self._info.get('license', 'unknown')

    @property
    def doi(self):
{'fraction_non_alphanumeric': 0.1257606490872211, 'fraction_numerical': 0.0008451656524678837, 'mean_word_length': 2.0005070993914806, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7217570', 'n_tokens_mistral': 1797, 'n_tokens_neox': 1723, 'n_words': 416}