<commit_before>#!/bin/bash

fail () {
    echo "$@" >&2
    exit 1
}

safe_config_val () {
    condor_ce_config_val $1 || fail "Failed to retrieve CE configuration value '$1'"
}

# Create a temporary accounting file name
today=$(date -u --date='00:00:00 today' +%s)
yesterday=$(date -u --date='00:00:00 yesterday' +%s)

OUTPUT_DIR="$(condor_ce_config_val APEL_OUTPUT_DIR)"
OUTPUT_FILE="$OUTPUT_DIR/batch-$(date -u --date='yesterday' +%Y%m%d )-$(hostname -s)"

if [ ! -d $OUTPUT_DIR ] || [ ! -w $OUTPUT_DIR ]; then
    echo "Cannot write to $OUTPUT_DIR"
    exit 1
fi

# Build the filter for the history command
CONSTR="EnteredCurrentStatus >= $yesterday && EnteredCurrentStatus < $today && RemoteWallclockTime !=0"

HISTORY_EXTRA_ARGS='-format "\n" EMPTY'
SCALING_ATTR=$(safe_config_val APEL_SCALING_ATTR)
[[ -z $SCALING_ATTR ]] || HISTORY_EXTRA_ARGS="-format \"%v|\" ${SCALING_ATTR} ${HISTORY_EXTRA_ARGS}"

TZ=GMT condor_history -constraint "$CONSTR" \
    -format "%s_$(safe_config_val APEL_BATCH_HOST)|" ClusterId \
    -format "%s|" Owner \
    -format "%d|" RemoteWallClockTime \
    -format "%d|" RemoteUserCpu \
    -format "%d|" RemoteSysCpu \
    -format "%d|" JobStartDate \
    -format "%d|" EnteredCurrentStatus \
    -format "%d|" ResidentSetSize_RAW \
    -format "%d|" ImageSize_RAW \
    -format "%d|" RequestCpus \
    ${HISTORY_EXTRA_ARGS} > $OUTPUT_FILE
<commit_msg>use 'fail' for $OUTPUT_DIR check also<commit_after>#!/bin/bash

fail () {
    echo "$@" >&2
    exit 1
}

safe_config_val () {
    condor_ce_config_val $1 || fail "Failed to retrieve CE configuration value '$1'"
}

# Create a temporary accounting file name
today=$(date -u --date='00:00:00 today' +%s)
yesterday=$(date -u --date='00:00:00 yesterday' +%s)

OUTPUT_DIR="$(condor_ce_config_val APEL_OUTPUT_DIR)"
OUTPUT_FILE="$OUTPUT_DIR/batch-$(date -u --date='yesterday' +%Y%m%d )-$(hostname -s)"

[[ -d $OUTPUT_DIR && -w $OUTPUT_DIR ]] || fail "Cannot write to $OUTPUT_DIR"

# Build the filter for the history command
CONSTR="EnteredCurrentStatus >= $yesterday && EnteredCurrentStatus < $today && RemoteWallclockTime !=0"

HISTORY_EXTRA_ARGS='-format "\n" EMPTY'
SCALING_ATTR=$(safe_config_val APEL_SCALING_ATTR)
[[ -z $SCALING_ATTR ]] || HISTORY_EXTRA_ARGS="-format \"%v|\" ${SCALING_ATTR} ${HISTORY_EXTRA_ARGS}"

TZ=GMT condor_history -constraint "$CONSTR" \
    -format "%s_$(safe_config_val APEL_BATCH_HOST)|" ClusterId \
    -format "%s|" Owner \
    -format "%d|" RemoteWallClockTime \
    -format "%d|" RemoteUserCpu \
    -format "%d|" RemoteSysCpu \
    -format "%d|" JobStartDate \
    -format "%d|" EnteredCurrentStatus \
    -format "%d|" ResidentSetSize_RAW \
    -format "%d|" ImageSize_RAW \
    -format "%d|" RequestCpus \
    ${HISTORY_EXTRA_ARGS} > $OUTPUT_FILE

{'fraction_non_alphanumeric': 0.16570188133140376, 'fraction_numerical': 0.012662807525325614, 'mean_word_length': 4.236742424242424, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5267978', 'n_tokens_mistral': 1072, 'n_tokens_neox': 1038, 'n_words': 234}