<commit_before>#!/bin/bash

if [[ $# != 2 ]]; then
    echo "usage: $0 email_address $0 file_with_ids_to_delete"
    echo "       \"email_address\" is where the report about successful deletions"
    echo "       and deletion failures will be  sent."
    echo "       \"file_with_ids_to_delete\" should contain the control numbers of"
    echo "       records we wish to delete.  One number per line."
    exit 1
fi

EMAIL_ADDRESS="$1"
INPUT_FILE="$2"
DELETION_LOG="/tmp/deletion.log"

> "$DELETION_LOG"
while read line; do
    if [[ ${#line} < 13 ]]; then
	echo "Weird short line: $line"
	continue
    fi

    record_type=${line:11:1}
    if [[ $record_type == 'A']]; then
        id=${line:12}
    elif [[ $record_type == '9' ]]; then
        id=${line:12:9}
    else
	continue
    fi

    result=$(curl "http://localhost:8080/solr/biblio/update?commit=true" \
             --data "<delete><query>$id</query></delete>" \
             --header 'Content-type:text/xml; charset=utf-8')
    if $(echo "$result" | grep -q '<int name="status">0</int>'); then
	echo "Successfully deleted control number $id" >> "$DELETION_LOG"
    else
	echo "Failed to delete control number $id" >> "$DELETION_LOG"
    fi
done < "$INPUT_FILE"

mailx -s "Import Deletion Log" "$1" < "$DELETION_LOG"<commit_msg>Delete records in batches of 10 to speed things up.<commit_after>#!/bin/bash
set -o errexit -o nounset
set -x

if [[ $# != 2 ]]; then
    echo "usage: $0 email_address $0 file_with_ids_to_delete"
    echo "       \"email_address\" is where the report about successful deletions"
    echo "       and deletion failures will be  sent."
    echo "       \"file_with_ids_to_delete\" should contain the control numbers of"
    echo "       records we wish to delete.  One number per line."
    exit 1
fi

EMAIL_ADDRESS="$1"
INPUT_FILE="$2"
DELETION_LOG="/tmp/deletion.log"
MAX_IDS_PER_CALL=10


function ExecCurl() {
    result=$(curl "http://localhost:8080/solr/biblio/update?commit=true" \
             --silent --show-error \
             --data "<delete><query>id:($id)</query></delete>" \
             --header 'Content-type:text/xml; charset=utf-8')
    if $(echo "$result" | grep -q '<int name="status">0</int>'); then
	echo "Called SOLR with ID's: $1"
    else
	echo "Failed to call SOLR! (id_list = $1)" >> "$DELETION_LOG"
	exit 1
    fi
}


> "$DELETION_LOG"
counter=0
id_list=""
while read line; do
    if [[ ${#line} < 13 ]]; then
	echo "Weird short line: $line"
	continue
    fi

    record_type=${line:11:1}
    if [[ $record_type == 'A' ]]; then
        id=${line:12}
    elif [[ $record_type == '9' ]]; then
        id=${line:12:9}
    else
	continue
    fi

    if [[ $counter == $MAX_IDS_PER_CALL ]]; then
        ExecCurl id_list
	counter=0
	id_list=""
    elif [[ $counter == 0 ]]; then
	id_list="$id"
	((++counter))
    else
	id_list="$id_list OR $id"
	((++counter))
    fi	  
done < "$INPUT_FILE"

if [[ "$id_list" != "" ]]; then
    ExecCurl id_list
fi

mailx -s "Import Deletion Log" "$1" < "$DELETION_LOG"
{'fraction_non_alphanumeric': 0.15897776302688352, 'fraction_numerical': 0.019249917026219716, 'mean_word_length': 2.8940568475452197, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1524104', 'n_tokens_mistral': 1104, 'n_tokens_neox': 1045, 'n_words': 291}