<commit_before>import json
from os import chdir, makedirs
from dask.delayed import delayed, value
from copy import deepcopy
from ..config import config as cfg
from ..tools.genrun import genrun

from importlib import import_module
metal = import_module(".metal", "nekpy.dask")

delayed = delayed(pure=True)

def configure(base, override, workdir):
    res = deepcopy(base)
    res.update(override)
    res["workdir"] = workdir
    if res["io_step"] == 0:
        res["io_step"] = 100000000
    if res["io_time"] == 0:
        res["io_time"] = res["end_time"]
    return res 

@delayed
def prepare(base, tusr, make=True):
    try:
        makedirs(base["workdir"])
    except OSError:
        pass
    chdir(base["workdir"]) 

    genrun(base["job_name"], base, tusr, do_make = make, makenek=cfg.makenek)
    return base<commit_msg>end time isn't always specified.<commit_after>import json
from os import chdir, makedirs
from dask.delayed import delayed, value
from copy import deepcopy
from ..config import config as cfg
from ..tools.genrun import genrun

from importlib import import_module
metal = import_module(".metal", "nekpy.dask")

delayed = delayed(pure=True)

def configure(base, override, workdir):
    res = deepcopy(base)
    res.update(override)
    res["workdir"] = workdir
    if res["io_step"] == 0:
        res["io_step"] = 100000000
    if res["io_time"] == 0 and "end_time" in res:
        res["io_time"] = res["end_time"]
    return res 

@delayed
def prepare(base, tusr, make=True):
    try:
        makedirs(base["workdir"])
    except OSError:
        pass
    chdir(base["workdir"]) 

    genrun(base["job_name"], base, tusr, do_make = make, makenek=cfg.makenek)
    return base
{'fraction_non_alphanumeric': 0.12345679012345678, 'fraction_numerical': 0.012933568489124045, 'mean_word_length': 3.701657458563536, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '615065', 'n_tokens_mistral': 595, 'n_tokens_neox': 559, 'n_words': 157}