<commit_before>#!/bin/bash
#Add reference data terms 
set -o errexit -o nounset


VUFIND_HOME="/usr/local/vufind2"
SOLR_HOME="$VUFIND_HOME/solr"
SOLRJ_PARENT="$SOLR_HOME/vendor/dist"
SOLRJ_HOME="$SOLRJ_PARENT/solrj-lib"
JETTY_LIB="$SOLR_HOME/vendor/server/solr-webapp/webapp/WEB-INF/lib"
TEMP_SOLRTEST="/home/qubhw01/solrjtest"
EXTLIB="$SOLR_HOME/vendor/server/lib/"
CLASSPATH="$SOLRJ_PARENT/*:$JETTY_LIB/*:$SOLRJ_HOME/*:$EXTLIB/*:$TEMP_SOLRTEST/*:."

echo $CLASSPATH

if [ $# != 2 ]; then
    echo "usage: $0 reference_list_file output_dir"
    exit 1;
fi

reffile=$1
outputdir=$2

cat $reffile | awk --file rewrite_query_expression.awk | xargs -0 --max-args=2 --max-procs=20 java -cp $CLASSPATH SolrJTest $outputdir
<commit_msg>Sorted output file is now generated<commit_after>#!/bin/bash
#Add reference data terms 
set -o errexit -o nounset


VUFIND_HOME="/usr/local/vufind2"
UBTOOLS_HOME="/usr/local/ub_tools"
SOLR_HOME="$VUFIND_HOME/solr"
SOLRJ_PARENT="$SOLR_HOME/vendor/dist"
SOLRJ_HOME="$SOLRJ_PARENT/solrj-lib"
JETTY_LIB="$SOLR_HOME/vendor/server/solr-webapp/webapp/WEB-INF/lib"
QUERYREF_HOME="$UBTOOLS_HOME/solrj_apps"
EXTLIB="$SOLR_HOME/vendor/server/lib/ext"
#RESOURCES="$SOLR_HOME/vendor/server/resources"
RESOURCES="$VUFIND_HOME/import"
CLASSPATH="$SOLRJ_PARENT/*:$JETTY_LIB/*:$SOLRJ_HOME/*:$EXTLIB/*:$QUERYREF_HOME/*:$RESOURCES/*:./*"

QUERYREF_MAIN=de.unituebingen.ub.ubtools.solrj_apps.QueryRefTermIds

echo $CLASSPATH

if [ $# != 2 ]; then
    echo "usage: $0 reference_list_file output_dir"
    exit 1;
fi

reffile=$1
outputdir=$2

#Query all matching IDs from Solr and write files for each term respectively
cat $reffile | awk --file rewrite_query_expression.awk | xargs -0 --max-args=2 --max-procs=8 java -cp $CLASSPATH  $QUERYREF_MAIN $outputdir

# Create a with with a list of IDs and all matching reference terms
cat $outputir/* | sort -k1 > $outputdir/UNIFIED
awk -F "|" 's != $1 || NR ==1{s=$1;if(p){print p};p=$0;next} {sub($1,"",$0);p=p""$0;}END{print p}' < $outputdir/UNIFIED > $outputdir/MERGED




{'fraction_non_alphanumeric': 0.16132215096201283, 'fraction_numerical': 0.014306857424765663, 'mean_word_length': 5.584415584415584, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6592633', 'n_tokens_mistral': 900, 'n_tokens_neox': 842, 'n_words': 145}