<commit_before>#!/bin/bash
set -o errexit -o nounset -o pipefail

if [ $# != 2 ]; then
    echo "usage: $0 ptsem-XX.zip output.mrc"
    exit 1
fi

archive_file="$1"
output_file="$2"
archive_contents=$(7z l ${archive_file} | grep 'xml$' | awk '{ print $6 }')
TOOL_BASE_PATH="/usr/local/ub_tools/cpp/princeton_tools"
XSLT_FILE="MODS3-7_MARC21slim_XSLT1-0.xsl"
PPN_BASE=$(echo ${archive_file} | sed -r -e 's/ptsem-([[:digit:]]+)\.zip/\1/')

> ${output_file}
echo '<?xml version="1.0" encoding="UTF-8"?>' >> ${output_file}
echo '<collection>' >> ${output_file}
i=0
for content in ${archive_contents}; do
     i=$((i+1))
     ppn=$(printf "${PPN_BASE}_%06d" ${i})
     echo ${ppn}
     echo ${content}
     7z x -so ${archive_file} ${content} | \
         xmlstarlet ed -d '//_:subject[@authority="aat" or @authority="local" or @authority="rbgenr"]' | \
         xmlstarlet tr ${TOOL_BASE_PATH}/${XSLT_FILE} | \
         xmlstarlet ed -O -a //marc:leader -t elem -n 'marc:controlfield' -v "${ppn}"  \
                    --var new_node '$prev' -i '$new_node' -t attr -n "tag" -v "001" | \
         sed -r -e 's/(<[/]?)marc:/\1/g' | \
         sed -r -e 's/(<record).*>/\1>/g' >> ${output_file}
#     if [[ ${i} -gt 20 ]]; then
#         break
#     fi
done
echo '</collection>' >> ${output_file}<commit_msg>Introduce skip list for apparently wrong records<commit_after>#!/bin/bash
set -o errexit -o nounset -o pipefail

if [ $# != 2 ]; then
    echo "usage: $0 ptsem-XX.zip output.mrc"
    exit 1
fi

archive_file="$1"
output_file="$2"
archive_contents=$(7z l ${archive_file} | grep 'xml$' | awk '{ print $6 }')
TOOL_BASE_PATH="/usr/local/ub_tools/cpp/princeton_tools"
XSLT_FILE="MODS3-7_MARC21slim_XSLT1-0.xsl"
PPN_BASE=$(echo ${archive_file} | sed -r -e 's/ptsem-([[:digit:]]+)\.zip/\1/')
SKIP_LIST=('annbi00bibl.xml')

> ${output_file}
echo '<?xml version="1.0" encoding="UTF-8"?>' >> ${output_file}
echo '<collection>' >> ${output_file}
i=0
for content in ${archive_contents}; do
     i=$((i+1))
     ppn=$(printf "${PPN_BASE}_%06d" ${i})
     echo ${ppn}
     echo ${content}
     #c.f. https://stackoverflow.com/questions/3685970/check-if-a-bash-array-contains-a-value
     if [[ " ${SKIP_LIST[*]} " =~ " ${content} " ]]; then
         echo "Skipping ${content}"
         continue
     fi
     7z x -so ${archive_file} ${content} | \
         xmlstarlet ed -d '//_:subject[@authority="aat" or @authority="local" or @authority="rbgenr"]' | \
         xmlstarlet tr ${TOOL_BASE_PATH}/${XSLT_FILE} | \
         xmlstarlet ed -O -a //marc:leader -t elem -n 'marc:controlfield' -v "${ppn}"  \
                    --var new_node '$prev' -i '$new_node' -t attr -n "tag" -v "001" | \
         sed -r -e 's/(<[/]?)marc:/\1/g' | \
         sed -r -e 's/(<record).*>/\1>/g' >> ${output_file}
#     if [[ ${i} -gt 20 ]]; then
#         break
#     fi
done
echo '</collection>' >> ${output_file}
{'fraction_non_alphanumeric': 0.2171349288935137, 'fraction_numerical': 0.023239680887963927, 'mean_word_length': 2.9025710419485793, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 2, '>': 31, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 11}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '527257', 'n_tokens_mistral': 1180, 'n_tokens_neox': 1132, 'n_words': 253}