<commit_before>#! /usr/bin/env sh
set -eux

if nvidia-smi --query-gpu=count; then
    RUNTIME='--runtime=nvidia'
    MARKS='gpu'
else
    RUNTIME=
    MARKS='not gpu and not slow'
fi

docker run ${RUNTIME} --interactive --rm \
       --volume $(realpath .):/mnt --workdir /mnt \
       --env MPLBACKEND=agg \
       hakuyume/chainercv:chainer${CHAINER}-devel \
       sh -ex << EOD
pip${PYTHON} install --user -e .
mpiexec -n 2 --allow-run-as-root \
        python${PYTHON} -m pytest --color=no -m '${MARKS}' tests
EOD<commit_msg>split w/o mpi tests and w/ mpi tests<commit_after>#! /usr/bin/env sh
set -eux

if nvidia-smi --query-gpu=count; then
    RUNTIME='--runtime=nvidia'
    MARKS='gpu'
else
    RUNTIME=
    MARKS='not gpu and not slow'
fi

docker run ${RUNTIME} --interactive --rm \
       --volume $(realpath .):/mnt --workdir /mnt \
       --env MPLBACKEND=agg \
       hakuyume/chainercv:chainer${CHAINER}-devel \
       sh -ex << EOD
pip${PYTHON} install --user -e .
python${PYTHON} -m pytest --color=no -m '${MARKS} and not mpi' tests
mpiexec -n 2 --allow-run-as-root \
        python${PYTHON} -m pytest --color=no -m '${MARKS} and mpi' tests
EOD
{'fraction_non_alphanumeric': 0.16637931034482759, 'fraction_numerical': 0.0017241379310344827, 'mean_word_length': 2.43491124260355, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2440985', 'n_tokens_mistral': 477, 'n_tokens_neox': 457, 'n_words': 120}