<commit_before>#!/bin/bash
# Runs through the phases of the KrimDok MARC processing pipeline.
set -o errexit -o nounset

if [ -z "$VUFIND_HOME" ]; then
    VUFIND_HOME=/usr/local/vufind2
fi

if [ $# != 2 ]; then
    echo "usage: $0 GesamtTiteldaten-YYMMDD.mrc" "Normdaten-YYMMDD.mrc"
    exit 1
fi

if [[ ! "$1" =~ GesamtTiteldaten-[0-9][0-9][0-9][0-9][0-9][0-9].mrc ]]; then
    echo 'Die Gesamttiteldatendatei entspricht nicht dem Muster GesamtTiteldaten-[0-9][0-9][0-9][0-9][0-9][0-9].mrc!'
    exit 1
fi


# Determines the embedded date of the files we're processing:
date=$(echo $(echo "$1" | cut -d- -f 2) | cut -d. -f1)


if [[ "$2" != "Normdaten-${date}.mrc" ]]; then
    echo "Authority data file must be named \"$2\"\ to match the bibliographic file name!"
    exit 1
fi


function StartPhase {
    if [ -z ${PHASE+x} ]; then
        PHASE=0
    else
        ((++PHASE))
    fi
    START=$(date +%s.%N)
    echo "*** Phase $PHASE: $1 - $(date) ***" | tee --append "${log}"
}


# Call with "CalculateTimeDifference $start $end".
# $start and $end have to be in seconds.
# Returns the difference in fractional minutes as a string.
function CalculateTimeDifference {
    start=$1
    end=$2
    echo "scale=2;($end - $start)/60" | bc --mathlib
}


function EndPhase {
    PHASE_DURATION=$(CalculateTimeDifference $START $(date +%s.%N))
    echo -e "Done after ${PHASE_DURATION} minutes.\n" | tee --append "${log}"
}


# Set up the log file:
logdir=/var/log/krimdok
log="${logdir}/krimdok_marc_pipeline.log"
rm -f "${log}"


OVERALL_START=$(date +%s.%N)


StartPhase "Filter out Records of Other Institutions"
delete_unused_local_data GesamtTiteldaten-"${date}".mrc \
                         GesamtTiteldaten-post-phase"$PHASE"-"${date}".mrc \
                         >> "${log}" 2>&1
EndPhase


StartPhase "Normalise URL's"
normalise_urls GesamtTiteldaten-post-phase"$((PHASE-1))"-"${date}".mrc \
               GesamtTiteldaten-post-phase"$PHASE"-"${date}".mrc >> "${log}" 2>&1
EndPhase


StartPhase "Add Author Synonyms from Norm Data"
add_author_synonyms GesamtTiteldaten-post-phase"$((PHASE-1))"-"${date}".mrc Normdaten-"${date}".mrc \
                    GesamtTiteldaten-post-phase"$PHASE"-"${date}".mrc >> "${log}" 2>&1
EndPhase


StartPhase "Add PDA Fields to Some Records"
krimdok_flag_pda_records GesamtTiteldaten-post-phase"$((PHASE-1))"-"${date}".mrc \
                         GesamtTiteldaten-post-phase"$PHASE"-"${date}".mrc >> "${log}" 2>&1
EndPhase

<commit_msg>Adding missing argument for krimdok_marc_pipeline.sh<commit_after>#!/bin/bash
# Runs through the phases of the KrimDok MARC processing pipeline.
set -o errexit -o nounset

if [ -z "$VUFIND_HOME" ]; then
    VUFIND_HOME=/usr/local/vufind2
fi

if [ $# != 2 ]; then
    echo "usage: $0 GesamtTiteldaten-YYMMDD.mrc" "Normdaten-YYMMDD.mrc"
    exit 1
fi

if [[ ! "$1" =~ GesamtTiteldaten-[0-9][0-9][0-9][0-9][0-9][0-9].mrc ]]; then
    echo 'Die Gesamttiteldatendatei entspricht nicht dem Muster GesamtTiteldaten-[0-9][0-9][0-9][0-9][0-9][0-9].mrc!'
    exit 1
fi


# Determines the embedded date of the files we're processing:
date=$(echo $(echo "$1" | cut -d- -f 2) | cut -d. -f1)


if [[ "$2" != "Normdaten-${date}.mrc" ]]; then
    echo "Authority data file must be named \"$2\"\ to match the bibliographic file name!"
    exit 1
fi


function StartPhase {
    if [ -z ${PHASE+x} ]; then
        PHASE=0
    else
        ((++PHASE))
    fi
    START=$(date +%s.%N)
    echo "*** Phase $PHASE: $1 - $(date) ***" | tee --append "${log}"
}


# Call with "CalculateTimeDifference $start $end".
# $start and $end have to be in seconds.
# Returns the difference in fractional minutes as a string.
function CalculateTimeDifference {
    start=$1
    end=$2
    echo "scale=2;($end - $start)/60" | bc --mathlib
}


function EndPhase {
    PHASE_DURATION=$(CalculateTimeDifference $START $(date +%s.%N))
    echo -e "Done after ${PHASE_DURATION} minutes.\n" | tee --append "${log}"
}


# Set up the log file:
logdir=/var/log/krimdok
log="${logdir}/krimdok_marc_pipeline.log"
rm -f "${log}"


OVERALL_START=$(date +%s.%N)


StartPhase "Filter out Records of Other Institutions"
delete_unused_local_data GesamtTiteldaten-"${date}".mrc \
                         GesamtTiteldaten-post-phase"$PHASE"-"${date}".mrc \
                         >> "${log}" 2>&1
EndPhase


StartPhase "Normalise URL's"
normalise_urls GesamtTiteldaten-post-phase"$((PHASE-1))"-"${date}".mrc \
               GesamtTiteldaten-post-phase"$PHASE"-"${date}".mrc >> "${log}" 2>&1
EndPhase


StartPhase "Add Author Synonyms from Norm Data"
add_author_synonyms GesamtTiteldaten-post-phase"$((PHASE-1))"-"${date}".mrc Normdaten-"${date}".mrc \
                    GesamtTiteldaten-post-phase"$PHASE"-"${date}".mrc >> "${log}" 2>&1
EndPhase


StartPhase "Add PDA Fields to Some Records"
krimdok_flag_pda_records 3 \
                         GesamtTiteldaten-post-phase"$((PHASE-1))"-"${date}".mrc \
                         GesamtTiteldaten-post-phase"$PHASE"-"${date}".mrc >> "${log}" 2>&1
EndPhase


{'fraction_non_alphanumeric': 0.17486122125297382, 'fraction_numerical': 0.021609833465503568, 'mean_word_length': 3.1901993355481726, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '214827', 'n_tokens_mistral': 2000, 'n_tokens_neox': 1918, 'n_words': 435}