<commit_before>#exit on any failure
set -e

#!/bin/bash
ENV=$1
P=$2
GIT_CREDENTIALS=$3
VERSION=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep ^0.`

# we use settings from a provate github repo with secret passwords
if [ -d "gbif-configuration" ]; then
  #If gitrepo exists, update the repositories
  echo "Updating local gbif-configuration Git repository"
  cd gbif-configuration
  git pull --all
  cd ..
else
  echo "Cloning Git repository gbif-configuration"
  git clone https://$GIT_CREDENTIALS@github.com/gbif/gbif-configuration
fi

echo "Assembling workflow jar for $ENV"
mvn --settings gbif-configuration/occurrence-download-workflow/$VERSION/profiles.xml -P$P -DskipTests clean package assembly:single

echo "Copy workflow to hadoop"
hadoop fs -rm -r -skipTrash /occurrence-download/$ENV
hadoop fs -put target/oozie-workflow /occurrence-download/$ENV
echo "Copied to hadoop"<commit_msg>Use simpler OAuth token from github to retrieve configs<commit_after>#exit on any failure
set -e

#!/bin/bash
ENV=$1
P=$2
TOKEN=$3
VERSION=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep ^0.`

echo "Get latest workflow config profiles from github"
curl -H 'Authorization: token $TOKEN' -H 'Accept: application/vnd.github.v3.raw' -O -L https://api.github.com/repos/gbif/gbif-configuration/contents/occurrence-download-workflow/$VERSION/profiles.xml

echo "Assembling workflow jar for $ENV"
mvn --settings profiles.xml -P$P -DskipTests clean package assembly:single

echo "Copy workflow to hadoop"
hadoop fs -rm -r -skipTrash /occurrence-download/$ENV
hadoop fs -put target/oozie-workflow /occurrence-download/$ENV
echo "Copied to hadoop"
{'fraction_non_alphanumeric': 0.1125866050808314, 'fraction_numerical': 0.008660508083140877, 'mean_word_length': 4.7959866220735785, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 3}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2751474', 'n_tokens_mistral': 591, 'n_tokens_neox': 557, 'n_words': 167}