<commit_before>#!/bin/bash
set -o errexit -o nounset

if [ $# != 0 ]; then
    echo "Usage: $0"
    exit 1
fi

#krimdok_nacjd_2220801_001.xml
declare -r BSZ_FILENAME="krimdok_nacjd_$(date +%Y%m%d)_001"
declare -r DOWNLOAD_DIR=/tmp/NACJD/$(date +%Y%m%d_%H%M%S)
declare -r DOWNLOAD_FILE="$DOWNLOAD_DIR/$BSZ_FILENAME.json"
declare -r CONVERT_FILE="$DOWNLOAD_DIR/$BSZ_FILENAME.xml"
mkdir -p "$DOWNLOAD_DIR"

# Use empty MRC file as dummy input.
# On consecutive downloads we want to use the real KrimDok GesamtTiteldaten... file here.
declare -r EMPTY_MRC_FILE="$DOWNLOAD_DIR/empty.mrc"
touch "$EMPTY_MRC_FILE"

echo "Downloading data"
nacjd get_full "$EMPTY_MRC_FILE" "$DOWNLOAD_FILE"

echo "Generating statistics (optional)"
nacjd get_statistics "$DOWNLOAD_FILE"

echo "Converting to MRC"
nacjd convert_JSON_to_MARC "$DOWNLOAD_FILE" "$CONVERT_FILE"

echo "Upload to BSZ"
#upload_to_bsz_ftp_server.py "$CONVERT_FILE" /pub/UBTuebingen_Default_Test/<commit_msg>nacjd_pipeline.sh: Disabled log message for inactive upload command<commit_after>#!/bin/bash
set -o errexit -o nounset

if [ $# != 0 ]; then
    echo "Usage: $0"
    exit 1
fi

#krimdok_nacjd_2220801_001.xml
declare -r BSZ_FILENAME="krimdok_nacjd_$(date +%Y%m%d)_001"
declare -r DOWNLOAD_DIR=/tmp/NACJD/$(date +%Y%m%d_%H%M%S)
declare -r DOWNLOAD_FILE="$DOWNLOAD_DIR/$BSZ_FILENAME.json"
declare -r CONVERT_FILE="$DOWNLOAD_DIR/$BSZ_FILENAME.xml"
mkdir -p "$DOWNLOAD_DIR"

# Use empty MRC file as dummy input.
# On consecutive downloads we want to use the real KrimDok GesamtTiteldaten... file here.
declare -r EMPTY_MRC_FILE="$DOWNLOAD_DIR/empty.mrc"
touch "$EMPTY_MRC_FILE"

echo "Downloading data"
nacjd get_full "$EMPTY_MRC_FILE" "$DOWNLOAD_FILE"

echo "Generating statistics (optional)"
nacjd get_statistics "$DOWNLOAD_FILE"

echo "Converting to MRC"
nacjd convert_JSON_to_MARC "$DOWNLOAD_FILE" "$CONVERT_FILE"

#echo "Upload to BSZ"
#upload_to_bsz_ftp_server.py "$CONVERT_FILE" /pub/UBTuebingen_Default_Test/
{'fraction_non_alphanumeric': 0.12150482968988308, 'fraction_numerical': 0.01626842907981698, 'mean_word_length': 6.00355871886121, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3283107', 'n_tokens_mistral': 844, 'n_tokens_neox': 791, 'n_words': 180}