<commit_before>#!/bin/bash
#
# $Id$
#
# Some of the initial setup that's common to the mk*.sh scripts
#
######################################################################

START_TIME=`date +%s`
START_DATE=`date '+%Y-%m-%d'`

# Set trapping for killing of running builds
unset PIDLIST
# On trap, exit with untrapped error code or else get infinite loop
trap 'TERMINATE_REQUESTED=true TERMINATE_ERROR_MSG=Killed. cleanup; exit 3' 1 2 15
trap -p >/tmp/traps$$.txt
traps=`sed 's/trap --//' </tmp/traps$$.txt | tr -d '\n'`
echo "Traps are $traps."
rm /tmp/traps$$.txt
export TERMINATE_REQUESTED=false
export TERMINATE_ERROR_MSG=

# Remove old indicators, provide new
rm -f $BILDER_LOGDIR/$BILDER_NAME.end
hostname >$BILDER_LOGDIR/$BILDER_NAME.host

######################################################################
#
# Allow systems first to set any flags.  Should not override option
# settings or environment.  May reset path, so must be done before
# svn invoked.
#
######################################################################

# Get machine hostname, directories, before, so machine files can
# host-specific cases in them
source $BILDER_DIR/runnr/runnrfcns.sh
runnrGetHostVars
techo "Working on $FQHOSTNAME of $BLDRHOSTID with RUNNRSYSTEM = $RUNNRSYSTEM."
techo "uname = `uname -a`."

# Make sure we have a machine file for Windows
if test -z "$MACHINE_FILE" && [[ `uname` =~ CYGWIN ]]; then
  MACHINE_FILE=cygwin.vs9
fi

# Set variables before sourcing machine file
REPLACE_COMPILERS=${REPLACE_COMPILERS:-"false"}

# Get machine specific variables
if test -n "$CC"; then
  techo "WARNING: [bildinit.sh] CC set to $CC before sourcing machine file"
fi
techo "MACHINE_FILE = \"$MACHINE_FILE\".  FQMAILHOST = \"$FQMAILHOST\"."
if test -n "$MACHINE_FILE"; then
  mfilefound=false
  if test -n "$BILDER_CONFDIR" -a -f $BILDER_CONFDIR/machines/$MACHINE_FILE; then
    absmachfile=$BILDER_CONFDIR/machines/$MACHINE_FILE
    techo "Sourcing $absmachfile."
    source $absmachfile
    techo "$absmachfile sourced."
    mfilefound=true
  fi
  if test -f $BILDER_DIR/machines/$MACHINE_FILE; then
    absmachfile=$BILDER_DIR/machines/$MACHINE_FILE
    techo "Sourcing $absmachfile."
    source $absmachfile
    techo "$absmachfile sourced."
    mfilefound=true
  fi
  if ! $mfilefound; then
    if test -n "$BILDER_CONFDIR"; then
      techo "WARNING: [bildinit.sh] $MACHINE_FILE not found in $BILDER_CONFDIR or $BILDER_DIR."
    else
      techo "WARNING: [bildinit.sh] $MACHINE_FILE not found in $BILDER_DIR."
    fi
  fi
elif test -n "$BILDER_CONFDIR" -a -f $BILDER_CONFDIR/machines/$FQMAILHOST; then
  absmachfile=$BILDER_CONFDIR/machines/$FQMAILHOST
  techo "Sourcing $absmachfile."
  source $absmachfile
  techo "$absmachfile sourced."
elif test -f $BILDER_DIR/machines/$FQMAILHOST; then
  absmachfile=$BILDER_DIR/machines/$FQMAILHOST
  techo "Sourcing $absmachfile."
  source $absmachfile
  techo "$absmachfile sourced."
else
  techo "No host specific variables file to source."
fi
# techo "CXXFLAGS = $CXXFLAGS"
# techo "Quitting in bildinit.sh."; exit

# Set BILDER_SVN before going further
BILDER_SVN=${BILDER_SVN:-"`which svn`"}
if test -z "$BILDER_SVN"; then
  TERMINATE_ERROR_MSG="ERROR: [bildinit.sh] svn not found."
  cleanup
fi
export BILDER_SVN # Needed for setinstald.sh

# The BEN build is an additional build for back end nodes.
# On BGP, the back end build uses a different, 32-bit serial
#   compiler for serial packages like txphysics.  The parallel
#   uses the (only) parallel (32-bit) compilers.
# One systems with outboard Phi nodes, the ben build is an
#   additional build using the extra parallel compilers, even
#   on serial only packages, like txphysics.
if test -z "$HAVE_BEN_BUILDS"; then
  HAVE_BEN_BUILDS=false
  if test "$CONFIG_COMPILERS_BEN" != "$CONFIG_COMPILERS_SER" -a "$CONFIG_COMPILERS_BEN" != "$CONFIG_COMPILERS_PAR"; then
    HAVE_BEN_BUILDS=true
  elif test "$CONFIG_COMPFLAGS_BEN" != "$CONFIG_COMPFLAGS_SER" -a \
      "$CONFIG_COMPFLAGS_BEN" != "$CONFIG_COMPFLAGS_PAR"; then
    HAVE_BEN_BUILDS=true
  fi
fi

######################################################################
#
# Update if requested
#
######################################################################

SVNUP=${SVNUP:-"false"}
if $SVNUP; then
  cmd="bilderSvn up --accept postpone $PROJECT_DIR"
  techo "$cmd" | tee $BILDER_LOGDIR/svnup.out
  $cmd 1>>$BILDER_LOGDIR/svnup.out 2>&1
  if test -x $PROJECT_DIR/updaterepos.sh; then
    cmd="(cd $PROJECT_DIR; ./updaterepos.sh)"
    techo "$cmd" | tee $BILDER_LOGDIR/svnup.out
    eval "$cmd" 1>>$BILDER_LOGDIR/svnup.out 2>&1
  fi
fi

######################################################################
#
# Check for versions
# Jenkins uses svnkit, which checks projects using the windows svn
# Subsequent use of the cygwin svn then says any directory with a
# link in it has been modified.  So unibild defines BLDR_SVNVERSION,
# which is the path to the windows subversion.
#
######################################################################

export USE_2ND_SVNAUTH=false
export BLDR_SVNVERSION=${BLDR_SVNVERSION:-"svnversion"}
techo "PATH = $PATH."
techo "svnversion = `which svnversion`."
BILDER_VERSION=`bilderSvnversion $BILDER_DIR`
SVN_BLDRVERSION=`bilderSvn -q --version --quiet | tr -d '\r'`
techo "Subversion version = $SVN_BLDRVERSION."
case $SVN_BLDRVERSION in
  1.[8-9].*)
    techo "WARNING: [bildinit.sh] Subversion version $SVN_BLDRVERSION is too new to work with Jenkins.  Please have 1.6-7.x installed and fix your path."
    ;;
  1.[1-5].*)
    techo "WARNING: [bildinit.sh] Subversion version $SVN_BLDRVERSION is too old.  Lacks --trust-server-cert.  Please have 1.6-7.x installed and fix your path."
    ;;
esac
# techo exit; exit

# Get various URLs
PROJECT_SVN_URL=`bilderSvn info $PROJECT_DIR | grep ^URL: | sed -e 's/^URL: *//'`
BILDER_URL=`bilderSvn info $BILDER_DIR | grep ^URL: | sed -e 's/^URL: *//'`
BILDER_BRANCH=`echo $BILDER_URL | sed -e 's?^.*/bilder/??'`
# techo "NOTE: [bildinit.sh] BILDER_BRANCH = $BILDER_BRANCH."
BILDER_BRANCHSHORT=`echo $BILDER_BRANCH | sed -e 's?^.*/??'`
BILDER_CONFDIR=${BILDER_CONFDIR:-"$BILDER_DIR/runnr"}
techo "BILDER_CONFDIR = $BILDER_CONFDIR."
BILDERCONF_VERSION=`bilderSvnversion $BILDER_CONFDIR`
techo "BILDERCONF_VERSION = $BILDERCONF_VERSION."

# Determine the name of this package
if test -n "$BILDER_PACKAGE"; then
  techo "WARNING: BILDER_PACKAGE defined to be $BILDER_PACKAGE.  Will not be used."
fi
PROJECT_URL=`bilderSvn info $PROJECT_DIR | grep ^URL: | sed -e 's/^URL: *//'`
if test -z "$BILDER_PROJECT"; then
# Must allow for projects in a larger repo, like visitall
  BILDER_PROJECT=`echo $PROJECT_URL | sed -e 's?/trunk??' -e 's?/branches/.*$??' -e 's?/tags/.*$??' -e 's?^.*/??'`
fi
techo "BILDER_PROJECT = $BILDER_PROJECT."
export BILDER_PROJECT
PROJECT_BRANCH=`echo $PROJECT_URL | sed -e "s?^.*/$BILDER_PROJECT/??"`
techo "NOTE: [bildinit.sh] PROJECT_BRANCH = $PROJECT_BRANCH."

# Clean out artifacts
if test -n "$JENKINS_FSROOT"; then
  cmd="(cd $PROJECT_DIR; $PROJECT_DIR/bilder/jenkins/jenkinsclean.sh)"
  echo "$cmd"
  eval "$cmd"
fi

# Get the packages repos
getPkgRepos

# Variables in which we record any failures
unset pidsKilled
unset actionsRunning<commit_msg>Making svnpkgs topdir more universal<commit_after>#!/bin/bash
#
# $Id$
#
# Some of the initial setup that's common to the mk*.sh scripts
#
######################################################################

START_TIME=`date +%s`
START_DATE=`date '+%Y-%m-%d'`

# Set trapping for killing of running builds
unset PIDLIST
# On trap, exit with untrapped error code or else get infinite loop
trap 'TERMINATE_REQUESTED=true TERMINATE_ERROR_MSG=Killed. cleanup; exit 3' 1 2 15
trap -p >/tmp/traps$$.txt
traps=`sed 's/trap --//' </tmp/traps$$.txt | tr -d '\n'`
echo "Traps are $traps."
rm /tmp/traps$$.txt
export TERMINATE_REQUESTED=false
export TERMINATE_ERROR_MSG=

# Remove old indicators, provide new
rm -f $BILDER_LOGDIR/$BILDER_NAME.end
hostname >$BILDER_LOGDIR/$BILDER_NAME.host

######################################################################
#
# Allow systems first to set any flags.  Should not override option
# settings or environment.  May reset path, so must be done before
# svn invoked.
#
######################################################################

# Get machine hostname, directories, before, so machine files can
# host-specific cases in them
source $BILDER_DIR/runnr/runnrfcns.sh
runnrGetHostVars
techo "Working on $FQHOSTNAME of $BLDRHOSTID with RUNNRSYSTEM = $RUNNRSYSTEM."
techo "uname = `uname -a`."

# Make sure we have a machine file for Windows
if test -z "$MACHINE_FILE" && [[ `uname` =~ CYGWIN ]]; then
  MACHINE_FILE=cygwin.vs9
fi

# Set variables before sourcing machine file
REPLACE_COMPILERS=${REPLACE_COMPILERS:-"false"}

# Get machine specific variables
if test -n "$CC"; then
  techo "WARNING: [bildinit.sh] CC set to $CC before sourcing machine file"
fi
techo "MACHINE_FILE = \"$MACHINE_FILE\".  FQMAILHOST = \"$FQMAILHOST\"."
if test -n "$MACHINE_FILE"; then
  mfilefound=false
  if test -n "$BILDER_CONFDIR" -a -f $BILDER_CONFDIR/machines/$MACHINE_FILE; then
    absmachfile=$BILDER_CONFDIR/machines/$MACHINE_FILE
    techo "Sourcing $absmachfile."
    source $absmachfile
    techo "$absmachfile sourced."
    mfilefound=true
  fi
  if test -f $BILDER_DIR/machines/$MACHINE_FILE; then
    absmachfile=$BILDER_DIR/machines/$MACHINE_FILE
    techo "Sourcing $absmachfile."
    source $absmachfile
    techo "$absmachfile sourced."
    mfilefound=true
  fi
  if ! $mfilefound; then
    if test -n "$BILDER_CONFDIR"; then
      techo "WARNING: [bildinit.sh] $MACHINE_FILE not found in $BILDER_CONFDIR or $BILDER_DIR."
    else
      techo "WARNING: [bildinit.sh] $MACHINE_FILE not found in $BILDER_DIR."
    fi
  fi
elif test -n "$BILDER_CONFDIR" -a -f $BILDER_CONFDIR/machines/$FQMAILHOST; then
  absmachfile=$BILDER_CONFDIR/machines/$FQMAILHOST
  techo "Sourcing $absmachfile."
  source $absmachfile
  techo "$absmachfile sourced."
elif test -f $BILDER_DIR/machines/$FQMAILHOST; then
  absmachfile=$BILDER_DIR/machines/$FQMAILHOST
  techo "Sourcing $absmachfile."
  source $absmachfile
  techo "$absmachfile sourced."
else
  techo "No host specific variables file to source."
fi
# techo "CXXFLAGS = $CXXFLAGS"
# techo "Quitting in bildinit.sh."; exit

# Set BILDER_SVN before going further
BILDER_SVN=${BILDER_SVN:-"`which svn`"}
if test -z "$BILDER_SVN"; then
  TERMINATE_ERROR_MSG="ERROR: [bildinit.sh] svn not found."
  cleanup
fi
export BILDER_SVN # Needed for setinstald.sh

# The BEN build is an additional build for back end nodes.
# On BGP, the back end build uses a different, 32-bit serial
#   compiler for serial packages like txphysics.  The parallel
#   uses the (only) parallel (32-bit) compilers.
# One systems with outboard Phi nodes, the ben build is an
#   additional build using the extra parallel compilers, even
#   on serial only packages, like txphysics.
if test -z "$HAVE_BEN_BUILDS"; then
  HAVE_BEN_BUILDS=false
  if test "$CONFIG_COMPILERS_BEN" != "$CONFIG_COMPILERS_SER" -a "$CONFIG_COMPILERS_BEN" != "$CONFIG_COMPILERS_PAR"; then
    HAVE_BEN_BUILDS=true
  elif test "$CONFIG_COMPFLAGS_BEN" != "$CONFIG_COMPFLAGS_SER" -a \
      "$CONFIG_COMPFLAGS_BEN" != "$CONFIG_COMPFLAGS_PAR"; then
    HAVE_BEN_BUILDS=true
  fi
fi

######################################################################
#
# Update if requested
#
######################################################################

SVNUP=${SVNUP:-"false"}
if $SVNUP; then
  cmd="bilderSvn up --accept postpone $PROJECT_DIR"
  techo "$cmd" | tee $BILDER_LOGDIR/svnup.out
  $cmd 1>>$BILDER_LOGDIR/svnup.out 2>&1
  if test -x $PROJECT_DIR/updaterepos.sh; then
    cmd="(cd $PROJECT_DIR; ./updaterepos.sh)"
    techo "$cmd" | tee $BILDER_LOGDIR/svnup.out
    eval "$cmd" 1>>$BILDER_LOGDIR/svnup.out 2>&1
  fi
fi

######################################################################
#
# Check for versions
# Jenkins uses svnkit, which checks projects using the windows svn
# Subsequent use of the cygwin svn then says any directory with a
# link in it has been modified.  So unibild defines BLDR_SVNVERSION,
# which is the path to the windows subversion.
#
######################################################################

export USE_2ND_SVNAUTH=false
export BLDR_SVNVERSION=${BLDR_SVNVERSION:-"svnversion"}
techo "PATH = $PATH."
techo "svnversion = `which svnversion`."
BILDER_VERSION=`bilderSvnversion $BILDER_DIR`
SVN_BLDRVERSION=`bilderSvn -q --version --quiet | tr -d '\r'`
techo "Subversion version = $SVN_BLDRVERSION."
case $SVN_BLDRVERSION in
  1.[8-9].*)
    techo "WARNING: [bildinit.sh] Subversion version $SVN_BLDRVERSION is too new to work with Jenkins.  Please have 1.6-7.x installed and fix your path."
    ;;
  1.[1-5].*)
    techo "WARNING: [bildinit.sh] Subversion version $SVN_BLDRVERSION is too old.  Lacks --trust-server-cert.  Please have 1.6-7.x installed and fix your path."
    ;;
esac
# techo exit; exit

# Get various URLs
PROJECT_SVN_URL=`bilderSvn info $PROJECT_DIR | grep ^URL: | sed -e 's/^URL: *//'`
BILDER_URL=`bilderSvn info $BILDER_DIR | grep ^URL: | sed -e 's/^URL: *//'`
BILDER_BRANCH=`echo $BILDER_URL | sed -e 's?^.*/bilder/??'`
# techo "NOTE: [bildinit.sh] BILDER_BRANCH = $BILDER_BRANCH."
BILDER_BRANCHSHORT=`echo $BILDER_BRANCH | sed -e 's?^.*/??'`
BILDER_CONFDIR=${BILDER_CONFDIR:-"$BILDER_DIR/runnr"}
techo "BILDER_CONFDIR = $BILDER_CONFDIR."
BILDERCONF_VERSION=`bilderSvnversion $BILDER_CONFDIR`
techo "BILDERCONF_VERSION = $BILDERCONF_VERSION."

# Determine the name of this package
if test -n "$BILDER_PACKAGE"; then
  techo "WARNING: BILDER_PACKAGE defined to be $BILDER_PACKAGE.  Will not be used."
fi
PROJECT_URL=`bilderSvn info $PROJECT_DIR | grep ^URL: | sed -e 's/^URL: *//'`
if test -z "$BILDER_PROJECT"; then
# Must allow for projects in a larger repo, like visitall
  BILDER_PROJECT=`echo $PROJECT_URL | sed -e 's?/trunk??' -e 's?/branches/.*$??' -e 's?/tags/.*$??' -e 's?^.*/??'`
fi
techo "BILDER_PROJECT = $BILDER_PROJECT."
export BILDER_PROJECT
PROJECT_BRANCH=`echo $PROJECT_URL | sed -e "s?^.*/$BILDER_PROJECT/??"`
techo "NOTE: [bildinit.sh] PROJECT_BRANCH = $PROJECT_BRANCH."

# Clean out artifacts
if test -n "$JENKINS_FSROOT"; then
  cmd="(cd $PROJECT_DIR; $PROJECT_DIR/bilder/jenkins/jenkinsclean.sh)"
  echo "$cmd"
  eval "$cmd"
fi

# Set default pkg topdir
SVNPKGS_TOPDIR=${SVNPKGS_TOPDIR:-"$JENKINS_FSROOT"}
SVNPKGS_TOPDIR=${SVNPKGS_TOPDIR:-"$GSCRATCH"}
SVNPKGS_TOPDIR=${SVNPKGS_TOPDIR:-"$SCRATCH"}
SVNPKGS_TOPDIR=${SVNPKGS_TOPDIR:-"$SCRATCH2"}
if test -d /scr_$UQHOSTNAME/$USER; then
  SVNPKGS_TOPDIR=${SVNPKGS_TOPDIR:-"/scr_$UQHOSTNAME/$USER"}
fi
SVNPKGS_TOPDIR=${SVNPKGS_TOPDIR:-"$HOME"}

# Get the packages repos
getPkgRepos

# Variables in which we record any failures
unset pidsKilled
unset actionsRunning
{'fraction_non_alphanumeric': 0.18069206947623537, 'fraction_numerical': 0.003971994075669854, 'mean_word_length': 4.932507987220447, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1138596', 'n_tokens_mistral': 5587, 'n_tokens_neox': 5440, 'n_words': 1441}