<commit_before>#!/bin/ksh

#/* Part of the Maestro sequencer software package.
# * Copyright (C) 2011-2015  Canadian Meteorological Centre
# *                          Environment Canada
# *
# * Maestro is free software; you can redistribute it and/or
# * modify it under the terms of the GNU Lesser General Public
# * License as published by the Free Software Foundation,
# * version 2.1 of the License.
# *
# * Maestro is distributed in the hope that it will be useful,
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# * Lesser General Public License for more details.
# *
# * You should have received a copy of the GNU Lesser General Public
# * License along with this library; if not, write to the
# * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# * Boston, MA 02111-1307, USA.
# */


# This script dumps all variables defined in the current
# shell before wrapping a call to task_setup.py.  It should
# be executed in the current shell using `. task_setup.ksh [ARGS]`
# where all ARGS are passed directly to the task setup utility.

# Define subroutine to check for matching arguments
checkarg()
{
  thisarg=$1; shift
  var=$1; shift
  patterns=$*
  found=0
  for pattern in ${patterns[*]}; do
    if [[ $(echo $thisarg | cut -c 1-${#pattern}) = $pattern ]] ; then
      eval $var=$(echo $thisarg | cut -c $((${#pattern}+1))-)
      found=1
    fi
  done
  unset thisarg
  unset var
  unset patterns
  unset pattern
}

# Define subroutine to run task_setup.py
launch_setup(){
   while [[ ! -f ${TS_setvar_file} ]] ; do
      echo "" >/dev/null  #super-awesome microsleep for event handler
   done
   if [[ ${TASK_SETUP_NOEXEC:-0} > 0 ]] ; then
      printf '** skipping TASK_SETUP execution with $TASK_SETUP_NOEXEC=%s **\n' ${TASK_SETUP_NOEXEC}
   else
      printf "** TASK_SETUP begins **\n"
      rm -f ${TS_abort_file}
      task_setup.py --environment ${TS_setvar_file} ${arglist} || touch ${TS_abort_file}
      printf "** TASK_SETUP ends **\n"
   fi
}

# Check command line arguments for the configuration file and task directory
. o.set_array.dot argv $*
TS_config_file=
TASK_BASEDIR=
arglist=
i=0
while [ -n "${argv[$i]}" ] ; do 
  arg=${argv[$i]}
  checkarg $arg 'TS_config_file' '--file=' '-f='
  if [[ ${found} == 1 ]] ; then arg='' ; fi
  if [[ $arg == '-f' || $arg == '--file' ]] ; then TS_config_file=${argv[$i+1]} ; arg='' ; fi
  checkarg ${arg} 'TASK_BASEDIR' '--base=' '-b='
  if [[ $arg == '-b' || $arg == '--base' ]] ; then TASK_BASEDIR=${argv[$i+1]} ; fi
  arglist="${arglist} ${arg}"
  i=$((i+1))
done
if [ -z "${TS_config_file}" ] ; then
  echo "WARNING: task_setup.ksh was unable to find a -f or --file argument"
fi
if [ ! -f "${TS_config_file}" ] ; then
  echo "WARNING: task_setup.ksh was unable to find specified configuration file "${TS_config_file}
fi
arglist="${arglist} ${TS_config_file}"

# Clean up shell and dot configuration file
export TASK_BASEDIR
unset argv
unset i

# Fork to avoid polluting the task_setup envrionment
TS_setvar_file=${TMPDIR:-/tmp}/task_setup_env$$
TS_abort_file=${TMPDIR:-/tmp}/task_setup_abort$$
launch_setup &
TS_setup_thread_pid=$!

# Obtain set namespace and write to a temporary file
if [[ -f ${TS_config_file} ]] ; then . ${TS_config_file} ; fi
unset TS_config_file

# Generate a temporary file containing all set variables
set >${TS_setvar_file}_locked
mv ${TS_setvar_file}_locked ${TS_setvar_file}

# Continue only after successful task_setup execution
wait ${TS_setup_thread_pid}
rm -f ${TS_setvar_file}
if [[ -f ${TS_abort_file} ]] ; then 
   rm -f ${TS_abort_file}
   exit 1
fi
rm -f ${TS_abort_file}

# Set up task environment
if [ -n "${TASK_BASEDIR}" ] ; then
  setup_truepath=${TASK_BASEDIR}/.setup/task_setup_truepath
  if [[ ! -x ${setup_truepath} ]] ; then
    setup_truepath=$(which true_path)
    printf '\tUnable to find true_path in %s ... will use %s\n' ${TASK_BASEDIR} ${setup_truepath}
  fi
  if [ -d ${TASK_BASEDIR}/bin ] ; then TASK_BIN=$(${setup_truepath} ${TASK_BASEDIR}/bin); export TASK_BIN ; fi
  if [ -d ${TASK_BASEDIR}/work ] ; then TASK_WORK=$(${setup_truepath} ${TASK_BASEDIR}/work) ; export TASK_WORK; fi
  if [ -d ${TASK_BASEDIR}/input ] ; then TASK_INPUT=$(${setup_truepath} ${TASK_BASEDIR}/input) ; export TASK_INPUT; fi
  if [ -d ${TASK_BASEDIR}/output ] ; then TASK_OUTPUT=$(${setup_truepath} ${TASK_BASEDIR}/output) ; export TASK_OUTPUT; fi
fi<commit_msg>Revert "Bug 7055 - Environment caching issue in how task_setup launches subprocesses"<commit_after>#!/bin/ksh

#/* Part of the Maestro sequencer software package.
# * Copyright (C) 2011-2015  Canadian Meteorological Centre
# *                          Environment Canada
# *
# * Maestro is free software; you can redistribute it and/or
# * modify it under the terms of the GNU Lesser General Public
# * License as published by the Free Software Foundation,
# * version 2.1 of the License.
# *
# * Maestro is distributed in the hope that it will be useful,
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# * Lesser General Public License for more details.
# *
# * You should have received a copy of the GNU Lesser General Public
# * License along with this library; if not, write to the
# * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# * Boston, MA 02111-1307, USA.
# */


# This script dumps all variables defined in the current
# shell before wrapping a call to task_setup.py.  It should
# be executed in the current shell using `. task_setup.ksh [ARGS]`
# where all ARGS are passed directly to the task setup utility.

# Define subroutine to check for matching arguments
checkarg()
{
  thisarg=$1; shift
  var=$1; shift
  patterns=$*
  found=0
  for pattern in ${patterns[*]}; do
    if [[ $(echo $thisarg | cut -c 1-${#pattern}) = $pattern ]] ; then
      eval $var=$(echo $thisarg | cut -c $((${#pattern}+1))-)
      found=1
    fi
  done
  unset thisarg
  unset var
  unset patterns
  unset pattern
}

# Check command line arguments for the configuration file and task directory
. o.set_array.dot argv $*
task_setup_cfgfile=
TASK_BASEDIR=
arglist=
i=0
while [ -n "${argv[$i]}" ] ; do 
  arg=${argv[$i]}
  checkarg $arg 'task_setup_cfgfile' '--file=' '-f='
  if [[ ${found} == 1 ]] ; then arg='' ; fi
  if [[ $arg == '-f' || $arg == '--file' ]] ; then task_setup_cfgfile=${argv[$i+1]} ; arg='' ; fi
  checkarg ${arg} 'TASK_BASEDIR' '--base=' '-b='
  if [[ $arg == '-b' || $arg == '--base' ]] ; then TASK_BASEDIR=${argv[$i+1]} ; fi
  arglist="${arglist} ${arg}"
  i=$((i+1))
done
if [ -z "${task_setup_cfgfile}" ] ; then
  echo "WARNING: task_setup.ksh was unable to find a -f or --file argument"
fi
if [ ! -f "${task_setup_cfgfile}" ] ; then
  echo "WARNING: task_setup.ksh was unable to find specified configuration file "${task_setup_cfgfile}
fi
arglist="${arglist} ${task_setup_cfgfile}"

# Clean up shell and dot configuration file
export TASK_BASEDIR
unset argv
unset i
if [[ -f ${task_setup_cfgfile} ]] ; then . ${task_setup_cfgfile} ; fi
unset task_setup_cfgfile

# Generate a temporary file containing all set variables
tmpfile=${TMPDIR:-/tmp}/task_setup_env$$
set >${tmpfile}

# Call task setup to generate task directory structure
if [[ ${TASK_SETUP_NOEXEC:-0} > 0 ]] ; then
    printf '** skipping TASK_SETUP execution with $TASK_SETUP_NOEXEC=%s **\n' ${TASK_SETUP_NOEXEC}
else
    printf "** TASK_SETUP begins **\n"
    task_setup.py --environment ${tmpfile} ${arglist} || exit 1
    printf "** TASK_SETUP ends **\n"
fi

# Export 
rm -f ${tmpfile}
if [ -n "${TASK_BASEDIR}" ] ; then
  setup_truepath=${TASK_BASEDIR}/.setup/task_setup_truepath
  if [[ ! -x ${setup_truepath} ]] ; then
    setup_truepath=$(which true_path)
    printf '\tUnable to find true_path in %s ... will use %s\n' ${TASK_BASEDIR} ${setup_truepath}
  fi
  if [ -d ${TASK_BASEDIR}/bin ] ; then TASK_BIN=$(${setup_truepath} ${TASK_BASEDIR}/bin); export TASK_BIN ; fi
  if [ -d ${TASK_BASEDIR}/work ] ; then TASK_WORK=$(${setup_truepath} ${TASK_BASEDIR}/work) ; export TASK_WORK; fi
  if [ -d ${TASK_BASEDIR}/input ] ; then TASK_INPUT=$(${setup_truepath} ${TASK_BASEDIR}/input) ; export TASK_INPUT; fi
  if [ -d ${TASK_BASEDIR}/output ] ; then TASK_OUTPUT=$(${setup_truepath} ${TASK_BASEDIR}/output) ; export TASK_OUTPUT; fi
fi
{'fraction_non_alphanumeric': 0.13938237748897103, 'fraction_numerical': 0.009538571598903063, 'mean_word_length': 3.9574468085106385, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '66106', 'n_tokens_mistral': 2902, 'n_tokens_neox': 2883, 'n_words': 930}