<commit_before>#!/bin/bash

set -e

COMMAND=${1:-dump}
CRON_SCHEDULE=${CRON_SCHEDULE:-0 1 * * *}
PREFIX=${PREFIX:-dump}
PGUSER=${PGUSER:-postgres}

if [[ "$COMMAND" == 'dump' ]]; then
    exec /dump.sh
elif [[ "$COMMAND" == 'dump-cron' ]]; then
    touch /var/log/cron.log
    CRON_ENV="PREFIX=$PREFIX\nPGUSER=$PGUSER"
    if [ -n "$PGPASSWORD" ]; then
        CRON_ENV="$CRON_ENV\nPGPASSWORD=$PGPASSWORD"
    fi
    echo -e "$CRON_ENV\n$CRON_SCHEDULE /dump.sh >> /var/log/cron.log 2>&1" | crontab -
    exec cron -f
else
    echo "Unknown command $COMMAND"
    echo "Available commands: dump, dump-cron"
    exit 1
fi<commit_msg>Send cron output to stdout<commit_after>#!/bin/bash

set -e

COMMAND=${1:-dump}
CRON_SCHEDULE=${CRON_SCHEDULE:-0 1 * * *}
PREFIX=${PREFIX:-dump}
PGUSER=${PGUSER:-postgres}

if [[ "$COMMAND" == 'dump' ]]; then
    exec /dump.sh
elif [[ "$COMMAND" == 'dump-cron' ]]; then
    touch /var/log/cron.log
    CRON_ENV="PREFIX=$PREFIX\nPGUSER=$PGUSER"
    if [ -n "$PGPASSWORD" ]; then
        CRON_ENV="$CRON_ENV\nPGPASSWORD=$PGPASSWORD"
    fi
    echo -e "$CRON_ENV\n$CRON_SCHEDULE /dump.sh >> /var/log/cron.log 2>&1"
    echo -e "$CRON_ENV\n$CRON_SCHEDULE /dump.sh >> /var/log/cron.log 2>&1" | crontab -
    cron
    tail -f /var/log/cron.log
else
    echo "Unknown command $COMMAND"
    echo "Available commands: dump, dump-cron"
    exit 1
fi
{'fraction_non_alphanumeric': 0.18248175182481752, 'fraction_numerical': 0.010218978102189781, 'mean_word_length': 3.324921135646688, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4748611', 'n_tokens_mistral': 562, 'n_tokens_neox': 563, 'n_words': 110}