<commit_before>#!/bin/sh

OUT_DIR=${OUT_DIR:-$PWD}
OUT_DIR=${1:-${OUT_DIR}}
echo $OUT_DIR

SRC_IMAGE=${SRC_IMAGE:-gcr.io/cafjs-k8/root-template:latest}
SRC_IMAGE=${2:-${SRC_IMAGE}}
echo $SRC_IMAGE

TEMPLATE_DIR=${TEMPLATE_DIR:-default}
TEMPLATE_DIR=${3:-${TEMPLATE_DIR}}
echo $TEMPLATE_DIR

mkdir -p ${OUT_DIR}

docker run -u $(id -u):$(id -g) -v ${OUT_DIR}:/out ${SRC_IMAGE} ${TEMPLATE_DIR} /out<commit_msg>Cleanup containers<commit_after>#!/bin/sh

OUT_DIR=${OUT_DIR:-$PWD}
OUT_DIR=${1:-${OUT_DIR}}
echo $OUT_DIR

SRC_IMAGE=${SRC_IMAGE:-gcr.io/cafjs-k8/root-template:latest}
SRC_IMAGE=${2:-${SRC_IMAGE}}
echo $SRC_IMAGE

TEMPLATE_DIR=${TEMPLATE_DIR:-default}
TEMPLATE_DIR=${3:-${TEMPLATE_DIR}}
echo $TEMPLATE_DIR

mkdir -p ${OUT_DIR}

docker run --rm -u $(id -u):$(id -g) -v ${OUT_DIR}:/out ${SRC_IMAGE} ${TEMPLATE_DIR} /out
{'fraction_non_alphanumeric': 0.2179176755447942, 'fraction_numerical': 0.009685230024213076, 'mean_word_length': 6.728971962616822, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6565721', 'n_tokens_mistral': 371, 'n_tokens_neox': 364, 'n_words': 55}