<commit_before>#!/bin/sh
# requirements: flac, flake, time, awk, stat, bc

# change location of binaries if necessary
flac="flac";
flake="../flake/flake";
wavinfo="./wavinfo";

enc="$flake $1 flake-";
dec="$flac -t flake-";

wavsize=$($wavinfo $1 | awk '/Data Size/ {print $3}');
playtime=$($wavinfo $1 | awk '/Playing Time/ {print $3}');
# not-as-accurate alternative for 16-bit 44100Hz stereo:
#wavsize=$(echo -e "$(stat -c %s $1) - 44" | bc);
#playtime=$(echo -e "scale=2\n$wavsize / (44100*4)" | bc)s;

echo "";
echo "flake:"
echo "";
echo "level   enc time    bytes    ratio   kbps    dec time";
echo "-----  ---------  --------   -----  ------  ---------";
for i in $(seq 0 8) ; do
    i0=$(printf "%02d" $i);
    time=$((time -p ${enc}$i0.flac -$i) 2>&1 | awk '/user/ {print $2}');
    csize=$(stat -c %s flake-$i0.flac);
    ratio=$(echo -e "scale=3\n$csize/$wavsize" | bc);
    kbps=$(echo -e "scale=1\n$csize*8/($playtime*1000)" | bc);
    dtime=$((time -p ${dec}$i0.flac) 2>&1 | awk '/user/ {print $2}');
    printf "%3d    %8ss  %8s   %5s  %6s  %8ss\n" $i $time $csize 0$ratio $kbps $dtime;
done;

echo "";
echo "flake (extended):";
echo "";
echo "level   enc time    bytes    ratio   kbps    dec time";
echo "-----  ---------  --------   -----  ------  ---------";
for i in $(seq 9 12) ; do
    i0=$(printf "%02d" $i);
    time=$((time -p ${enc}$i0.flac -$i) 2>&1 | awk '/user/ {print $2}');
    csize=$(stat -c %s flake-$i0.flac);
    ratio=$(echo -e "scale=3\n$csize/$wavsize" | bc);
    kbps=$(echo -e "scale=1\n$csize*8/($playtime*1000)" | bc);
    dtime=$((time -p ${dec}$i0.flac) 2>&1 | awk '/user/ {print $2}');
    printf "%3d    %8ss  %8s   %5s  %6s  %8ss\n" $i $time $csize 0$ratio $kbps $dtime;
done;
<commit_msg>update test script for new output option<commit_after>#!/bin/sh
# requirements: flac, flake, time, awk, stat, bc

# change location of binaries if necessary
flac="flac";
flake="../flake/flake";
wavinfo="./wavinfo";

enc="$flake $1 -o flake-";
dec="$flac -t flake-";

wavsize=$($wavinfo $1 | awk '/Data Size/ {print $3}');
playtime=$($wavinfo $1 | awk '/Playing Time/ {print $3}');
# not-as-accurate alternative for 16-bit 44100Hz stereo:
#wavsize=$(echo -e "$(stat -c %s $1) - 44" | bc);
#playtime=$(echo -e "scale=2\n$wavsize / (44100*4)" | bc)s;

echo "";
echo "flake:"
echo "";
echo "level   enc time    bytes    ratio   kbps    dec time";
echo "-----  ---------  --------   -----  ------  ---------";
for i in $(seq 0 8) ; do
    i0=$(printf "%02d" $i);
    time=$((time -p ${enc}$i0.flac -$i) 2>&1 | awk '/user/ {print $2}');
    csize=$(stat -c %s flake-$i0.flac);
    ratio=$(echo -e "scale=3\n$csize/$wavsize" | bc);
    kbps=$(echo -e "scale=1\n$csize*8/($playtime*1000)" | bc);
    dtime=$((time -p ${dec}$i0.flac) 2>&1 | awk '/user/ {print $2}');
    printf "%3d    %8ss  %8s   %5s  %6s  %8ss\n" $i $time $csize 0$ratio $kbps $dtime;
done;

echo "";
echo "flake (extended):";
echo "";
echo "level   enc time    bytes    ratio   kbps    dec time";
echo "-----  ---------  --------   -----  ------  ---------";
for i in $(seq 9 12) ; do
    i0=$(printf "%02d" $i);
    time=$((time -p ${enc}$i0.flac -$i) 2>&1 | awk '/user/ {print $2}');
    csize=$(stat -c %s flake-$i0.flac);
    ratio=$(echo -e "scale=3\n$csize/$wavsize" | bc);
    kbps=$(echo -e "scale=1\n$csize*8/($playtime*1000)" | bc);
    dtime=$((time -p ${dec}$i0.flac) 2>&1 | awk '/user/ {print $2}');
    printf "%3d    %8ss  %8s   %5s  %6s  %8ss\n" $i $time $csize 0$ratio $kbps $dtime;
done;

{'fraction_non_alphanumeric': 0.2702625570776256, 'fraction_numerical': 0.04509132420091324, 'mean_word_length': 2.5656154628687693, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6467806', 'n_tokens_mistral': 1539, 'n_tokens_neox': 1458, 'n_words': 389}