<commit_before>#!/usr/bin/env bash

COLOR_OFF='\e[0m'       # Text Reset
RED='\e[0;31m'          # Red
YELLOW='\e[0;33m'       # Yellow
GREEN='\e[0;32m'        # Green

RUNTIME="/tmp"
STORE="${RUNTIME}/store"

out() {
    echo -e "${YELLOW}:: $*${COLOR_OFF}"
}

success() {
    echo -e "${GREEN}>> $*${COLOR_OFF}"
}

err() {
    echo -e "${RED}!! $*${COLOR_OFF}"
}

imag-store() {
    local searchdir=$(dirname ${BASH_SOURCE[0]})/../target/debug/
    [[ -d $searchdir ]] || { err "FATAL: No directory $searchdir"; exit 1; }
    local bin=$(find $searchdir -iname imag-store -type f -executable)
    local flags="--debug --rtp $RUNTIME"
    out "Calling '$bin $flags $*'"
    $bin $flags $*
}

reset_store() {
    rm -r "${STORE}"
}
<commit_msg>Add silent() helper for calling without debugging output<commit_after>#!/usr/bin/env bash

COLOR_OFF='\e[0m'       # Text Reset
RED='\e[0;31m'          # Red
YELLOW='\e[0;33m'       # Yellow
GREEN='\e[0;32m'        # Green

RUNTIME="/tmp"
STORE="${RUNTIME}/store"

out() {
    [[ -z "$DEBUG_OUTPUT_OFF" ]] && echo -e "${YELLOW}:: $*${COLOR_OFF}"
}

success() {
    [[ -z "$DEBUG_OUTPUT_OFF" ]] && echo -e "${GREEN}>> $*${COLOR_OFF}"
}

err() {
    [[ -z "$DEBUG_OUTPUT_OFF" ]] && echo -e "${RED}!! $*${COLOR_OFF}"
}

silent() {
    DEBUG_OUTPUT_OFF=1 $*
}

imag-store() {
    local searchdir=$(dirname ${BASH_SOURCE[0]})/../target/debug/
    [[ -d $searchdir ]] || { err "FATAL: No directory $searchdir"; exit 1; }
    local bin=$(find $searchdir -iname imag-store -type f -executable)
    local flags="--debug --rtp $RUNTIME"
    out "Calling '$bin $flags $*'"
    $bin $flags $*
}

reset_store() {
    rm -r "${STORE}"
}

{'fraction_non_alphanumeric': 0.22975404919016196, 'fraction_numerical': 0.014997000599880024, 'mean_word_length': 3.0485436893203883, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5077346', 'n_tokens_mistral': 680, 'n_tokens_neox': 660, 'n_words': 141}