<commit_before>#!/bin/bash
# Runs through the phases of the Zotero Harvester delivery pipeline.
set -o errexit -o nounset


no_problems_found=1
function SendEmail {
    if [[ $no_problems_found -eq 0 ]]; then
        send_email --priority=low --sender="<EMAIL>" --recipients="$EMAIL_ADDRESS" \
                   --subject="$0 passed on $(hostname)" --message-body="No problems were encountered."
        exit 0
    else
        send_email --priority=high --sender="<EMAIL>" --recipients="$EMAIL_ADDRESS" \
                   --subject="$0 failed on $(hostname)" \
                   --message-body="Check the log file at /usr/local/var/log/tuefind/zts_harvester_delivery_pipeline.log for details."
        echo "*** ZTS_HARVESTER DELIVERY PIPELINE FAILED ***" | tee --append "${log}"
        exit 1
    fi
}
trap SendEmail EXIT


function Usage {
    echo "usage: $0 mode email"
    echo "       mode = TEST|LIVE"
    echo "       email = email address to which notifications are sent upon (un)successful completion of the delivery pipeline"
    exit 1
}


if [ $# != 2 ]; then
    Usage
fi


if [ "$1" != "TEST" ] && [ "$1" != "LIVE" ]; then
    echo "unknown mode '$1'"
    Usage
fi


readonly DELIVERY_MODE=$1
readonly EMAIL_ADDRESS=$2
readonly WORKING_DIRECTORY=/tmp/zts_harvester_delivery_pipeline

readonly HARVESTER_OUTPUT_DIRECTORY=$WORKING_DIRECTORY
readonly HARVESTER_OUTPUT_FILENAME=zts_harvester-$(date +%y%m%d).xml
readonly HARVESTER_CONFIG_FILE=/usr/local/var/lib/tuelib/zotero-enhancement-maps/zotero_harvester.conf

readonly DEST_DIR_LOCAL_TEST=/mnt/ZE020110/FID-Projekte/Default_Test/
readonly DEST_DIR_LOCAL_LIVE=/mnt/ZE020110/FID-Projekte/Default/
readonly DEST_DIR_REMOTE_TEST=/pub/UBTuebingen_Default_Test/
readonly DEST_DIR_REMOTE_LIVE=/pub/UBTuebingen_Default/



function StartPhase {
    if [ -z ${PHASE+x} ]; then
        PHASE=1
    else
        ((++PHASE))
    fi
    START=$(date +%s.%N)
    echo "*** Phase $PHASE: $1 - $(date) ***" | tee --append "${log}"
}


# Call with "CalculateTimeDifference $start $end".
# $start and $end have to be in seconds.
# Returns the difference in fractional minutes as a string.
function CalculateTimeDifference {
    start=$1
    end=$2
    echo "scale=2;($end - $start)/60" | bc --mathlib
}


function EndPhase {
    PHASE_DURATION=$(CalculateTimeDifference $START $(date +%s.%N))
    echo -e "Done after ${PHASE_DURATION} minutes.\n" | tee --append "${log}"
}


function EndPipeline {
    echo -e "\n\nPipeline done after $(CalculateTimeDifference $OVERALL_START $(date +%s.%N)) minutes." | tee --append "${log}"
    echo "*** ZTS_HARVESTER DELIVERY PIPELINE DONE ***" | tee --append "${log}"
    no_problems_found=0
    exit 0
}


# Set up the log file:
logdir=/usr/local/var/log/tuefind
log_filename=$(basename "$0")
log="${logdir}/${log_filename%.*}.log"
rm --force "${log}"

# Cleanup files/folders from a previous run
mkdir --parents $HARVESTER_OUTPUT_DIRECTORY
rm --recursive --force --dir $HARVESTER_OUTPUT_DIRECTORY/ixtheo
rm --recursive --force --dir $HARVESTER_OUTPUT_DIRECTORY/krimdok
rm --recursive --force --dir $HARVESTER_OUTPUT_DIRECTORY/ubtuebingen


OVERALL_START=$(date +%s.%N)
declare -a source_filepaths
declare -a dest_filepaths
declare -a dest_filepaths_local

StartPhase "Harvest URLs"
LOGGER_FORMAT=no_decorations,strip_call_site \
BACKTRACE=1 \
UTIL_LOG_DEBUG=true \
zotero_harvester --min-log-level=DEBUG \
             --output-directory=$HARVESTER_OUTPUT_DIRECTORY \
             --output-filename=$HARVESTER_OUTPUT_FILENAME \
             $HARVESTER_CONFIG_FILE \
             UPLOAD \
             $DELIVERY_MODE >> "${log}" 2>&1
EndPhase


StartPhase "Validate Generated Records"
cd $HARVESTER_OUTPUT_DIRECTORY
counter=0
shopt -s nullglob
for d in */ ; do
    d=${d%/}
    if [[ $d -ef $HARVESTER_OUTPUT_DIRECTORY ]]; then
        continue
    fi

    current_source_filepath=$HARVESTER_OUTPUT_DIRECTORY/$d/$HARVESTER_OUTPUT_FILENAME
    valid_records_output_filepath=$HARVESTER_OUTPUT_DIRECTORY/$d/zotero_${d}_$(date +%y%m%d)_001.xml  # we only deliver files once a day
    invalid_records_output_filepath=$HARVESTER_OUTPUT_DIRECTORY/$d/zotero_${d}_$(date +%y%m%d)_001_errors.xml
    LOGGER_FORMAT=no_decorations,strip_call_site \
    BACKTRACE=1 \
    UTIL_LOG_DEBUG=true \
    validate_harvested_records $current_source_filepath $valid_records_output_filepath \
                               $invalid_records_output_filepath $EMAIL_ADDRESS >> "${log}" 2>&1

    invalid_record_count=$(marc_size "$invalid_records_output_filepath")
    if [ "$invalid_record_count" != "0" ]; then
        if [ "$DELIVERY_MODE" = "TEST" ]; then
            cp $invalid_records_output_filepath $DEST_DIR_LOCAL_TEST
        elif [ "$DELIVERY_MODE" = "LIVE" ]; then
            cp $invalid_records_output_filepath $DEST_DIR_LOCAL_LIVE
        fi
    fi

    valid_record_count=$(marc_size "$valid_records_output_filepath")<commit_msg>zts_harvester_delivery_pipeline.sh: Change filenames of uploaded file from zotero_ixtheo_... to ixtheo_zotero_... as requested by the BSZ.<commit_after>#!/bin/bash
# Runs through the phases of the Zotero Harvester delivery pipeline.
set -o errexit -o nounset


no_problems_found=1
function SendEmail {
    if [[ $no_problems_found -eq 0 ]]; then
        send_email --priority=low --sender="<EMAIL>" --recipients="$EMAIL_ADDRESS" \
                   --subject="$0 passed on $(hostname)" --message-body="No problems were encountered."
        exit 0
    else
        send_email --priority=high --sender="<EMAIL>" --recipients="$EMAIL_ADDRESS" \
                   --subject="$0 failed on $(hostname)" \
                   --message-body="Check the log file at /usr/local/var/log/tuefind/zts_harvester_delivery_pipeline.log for details."
        echo "*** ZTS_HARVESTER DELIVERY PIPELINE FAILED ***" | tee --append "${log}"
        exit 1
    fi
}
trap SendEmail EXIT


function Usage {
    echo "usage: $0 mode email"
    echo "       mode = TEST|LIVE"
    echo "       email = email address to which notifications are sent upon (un)successful completion of the delivery pipeline"
    exit 1
}


if [ $# != 2 ]; then
    Usage
fi


if [ "$1" != "TEST" ] && [ "$1" != "LIVE" ]; then
    echo "unknown mode '$1'"
    Usage
fi


readonly DELIVERY_MODE=$1
readonly EMAIL_ADDRESS=$2
readonly WORKING_DIRECTORY=/tmp/zts_harvester_delivery_pipeline

readonly HARVESTER_OUTPUT_DIRECTORY=$WORKING_DIRECTORY
readonly HARVESTER_OUTPUT_FILENAME=zts_harvester-$(date +%y%m%d).xml
readonly HARVESTER_CONFIG_FILE=/usr/local/var/lib/tuelib/zotero-enhancement-maps/zotero_harvester.conf

readonly DEST_DIR_LOCAL_TEST=/mnt/ZE020110/FID-Projekte/Default_Test/
readonly DEST_DIR_LOCAL_LIVE=/mnt/ZE020110/FID-Projekte/Default/
readonly DEST_DIR_REMOTE_TEST=/pub/UBTuebingen_Default_Test/
readonly DEST_DIR_REMOTE_LIVE=/pub/UBTuebingen_Default/



function StartPhase {
    if [ -z ${PHASE+x} ]; then
        PHASE=1
    else
        ((++PHASE))
    fi
    START=$(date +%s.%N)
    echo "*** Phase $PHASE: $1 - $(date) ***" | tee --append "${log}"
}


# Call with "CalculateTimeDifference $start $end".
# $start and $end have to be in seconds.
# Returns the difference in fractional minutes as a string.
function CalculateTimeDifference {
    start=$1
    end=$2
    echo "scale=2;($end - $start)/60" | bc --mathlib
}


function EndPhase {
    PHASE_DURATION=$(CalculateTimeDifference $START $(date +%s.%N))
    echo -e "Done after ${PHASE_DURATION} minutes.\n" | tee --append "${log}"
}


function EndPipeline {
    echo -e "\n\nPipeline done after $(CalculateTimeDifference $OVERALL_START $(date +%s.%N)) minutes." | tee --append "${log}"
    echo "*** ZTS_HARVESTER DELIVERY PIPELINE DONE ***" | tee --append "${log}"
    no_problems_found=0
    exit 0
}


# Set up the log file:
logdir=/usr/local/var/log/tuefind
log_filename=$(basename "$0")
log="${logdir}/${log_filename%.*}.log"
rm --force "${log}"

# Cleanup files/folders from a previous run
mkdir --parents $HARVESTER_OUTPUT_DIRECTORY
rm --recursive --force --dir $HARVESTER_OUTPUT_DIRECTORY/ixtheo
rm --recursive --force --dir $HARVESTER_OUTPUT_DIRECTORY/krimdok
rm --recursive --force --dir $HARVESTER_OUTPUT_DIRECTORY/ubtuebingen


OVERALL_START=$(date +%s.%N)
declare -a source_filepaths
declare -a dest_filepaths
declare -a dest_filepaths_local

StartPhase "Harvest URLs"
LOGGER_FORMAT=no_decorations,strip_call_site \
BACKTRACE=1 \
UTIL_LOG_DEBUG=true \
zotero_harvester --min-log-level=DEBUG \
             --output-directory=$HARVESTER_OUTPUT_DIRECTORY \
             --output-filename=$HARVESTER_OUTPUT_FILENAME \
             $HARVESTER_CONFIG_FILE \
             UPLOAD \
             $DELIVERY_MODE >> "${log}" 2>&1
EndPhase


StartPhase "Validate Generated Records"
cd $HARVESTER_OUTPUT_DIRECTORY
counter=0
shopt -s nullglob
for d in */ ; do
    d=${d%/}
    if [[ $d -ef $HARVESTER_OUTPUT_DIRECTORY ]]; then
        continue
    fi

    current_source_filepath=$HARVESTER_OUTPUT_DIRECTORY/$d/$HARVESTER_OUTPUT_FILENAME

    # Output filenames MUST start with 'ixtheo_' or 'krimdok_', else BSZ will ignore it.
    # Also, since we deliver files only once a day, each file will be marked as "001".
    valid_records_output_filepath=$HARVESTER_OUTPUT_DIRECTORY/$d/${d}_zotero_$(date +%y%m%d)_001.xml
    invalid_records_output_filepath=$HARVESTER_OUTPUT_DIRECTORY/$d/${d}_zotero_$(date +%y%m%d)_001_errors.xml
    LOGGER_FORMAT=no_decorations,strip_call_site \
    BACKTRACE=1 \
    UTIL_LOG_DEBUG=true \
    validate_harvested_records $current_source_filepath $valid_records_output_filepath \
                               $invalid_records_output_filepath $EMAIL_ADDRESS >> "${log}" 2>&1

    invalid_record_count=$(marc_size "$invalid_records_output_filepath")
    if [ "$invalid_record_count" != "0" ]; then
        if [ "$DELIVERY_MODE" = "TEST" ]; then
            cp $invalid_records_output_filepath $DEST_DIR_LOCAL_TEST
        elif [ "$DELIVERY_MODE" = "LIVE" ]; then
            cp $invalid_records_output_filepath $DEST_DIR_LOCAL_LIVE
        fi
    fi

    valid_record_count=$(marc_size "$valid_records_output_filepath")
{'fraction_non_alphanumeric': 0.12252377179080824, 'fraction_numerical': 0.010202060221870047, 'mean_word_length': 3.5812159709618876, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '395784', 'n_tokens_mistral': 3630, 'n_tokens_neox': 3418, 'n_words': 751}