<commit_before>#exit on any failure
set -e

#!/bin/bash
P=$1
TOKEN=$2
#get the third parameter, if absent use a default of false
IS_SINGLE_SHARD=${3-'false'}

echo "Getting latest checklistbank-index-builder workflow properties file from github"
curl -s -H "Authorization: token $TOKEN" -H 'Accept: application/vnd.github.v3.raw' -O -L https://api.github.com/repos/gbif/gbif-configuration/contents/checklistbank-index-builder/$P.properties

if [ $IS_SINGLE_SHARD = true ] ; then
  echo -e "\nsolr.is_single_shard=true\n" >> $P.properties
else
  echo -e "\nsolr.is_single_shard=false\n" >> $P.properties
fi

#extract the oozie.url value from the properties file
oozie_url=`cat $P.properties| grep "oozie.url" | cut -d'=' -f2`

echo "Assembling jar for $ENV"

mvn -Poozie,$P clean package assembly:single
mvn -Psolr,$P package assembly:single

if hdfs dfs -test -d /checklistbank-index-builder-$P/; then
   echo "Removing content of current Oozie workflow directory"
   hdfs dfs -rm -f -r /checklistbank-index-builder-$P/*
else
   echo "Creating workflow directory"
   hdfs dfs -mkdir /checklistbank-index-builder-$P/
fi
echo "Copying new Oozie workflow to HDFS"
hdfs dfs -copyFromLocal target/oozie-workflow/* /checklistbank-index-builder-$P/
hdfs dfs -copyFromLocal $P.properties /checklistbank-index-builder-$P/lib/

echo "Executing Oozie workflow"
oozie job --oozie ${oozie_url} -config $P.properties -run
<commit_msg>Change index builder workflow installer to use maven properties from git<commit_after>#exit on any failure
set -e

#!/bin/bash
P=$1
TOKEN=$2
#get the third parameter, if absent use a default of false
IS_SINGLE_SHARD=${3-'false'}

echo "Getting latest checklistbank-index-builder workflow properties file from github"
curl -s -H "Authorization: token $TOKEN" -H 'Accept: application/vnd.github.v3.raw' -O -L https://api.github.com/repos/gbif/gbif-configuration/contents/checklistbank-index-builder/profiles.xml
curl -s -H "Authorization: token $TOKEN" -H 'Accept: application/vnd.github.v3.raw' -O -L https://api.github.com/repos/gbif/gbif-configuration/contents/checklistbank-index-builder/$P.properties

if [ $IS_SINGLE_SHARD = true ] ; then
  echo -e "\nsolr.is_single_shard=true\n" >> $P.properties
else
  echo -e "\nsolr.is_single_shard=false\n" >> $P.properties
fi

#extract the oozie.url value from the properties file
oozie_url=`cat $P.properties| grep "oozie.url" | cut -d'=' -f2`

echo "Assembling jar for $ENV"

mvn --settings profiles.xml -Poozie,$P clean package assembly:single
mvn --settings profiles.xml -Psolr,$P package assembly:single

if hdfs dfs -test -d /checklistbank-index-builder-$P/; then
   echo "Removing content of current Oozie workflow directory"
   hdfs dfs -rm -f -r /checklistbank-index-builder-$P/*
else
   echo "Creating workflow directory"
   hdfs dfs -mkdir /checklistbank-index-builder-$P/
fi
echo "Copying new Oozie workflow to HDFS"
hdfs dfs -copyFromLocal target/oozie-workflow/* /checklistbank-index-builder-$P/
hdfs dfs -copyFromLocal $P.properties /checklistbank-index-builder-$P/lib/

echo "Executing Oozie workflow"
oozie job --oozie ${oozie_url} -config $P.properties -run

{'fraction_non_alphanumeric': 0.12452229299363057, 'fraction_numerical': 0.0035031847133757963, 'mean_word_length': 4.3784246575342465, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5271154', 'n_tokens_mistral': 1116, 'n_tokens_neox': 1078, 'n_words': 314}