<commit_before>#!/usr/bin/env bash

# Where is the pcbsd-build program installed
PROGDIR="`realpath | sed 's|/scripts||g'`" ; export PROGDIR

# Source the config file
. ${PROGDIR}/freenas.cfg

cd ${PROGDIR}/scripts

# Source our functions
. ${PROGDIR}/scripts/functions.sh
. ${PROGDIR}/scripts/functions-tests.sh

# Look through the output log and try to determine the failure
parse_checkout_error()
{
  ### TODO - Add error detection of checkout failures
  echo '' > ${LOUT}
}

# Look through the output log and try to determine the failure
parse_build_error()
{
  echo '' > ${LOUT}
  export TESTSTDERR=${LOUT}

  # Look for some of the common error messages

  # port failed to compile
  grep -q "ERROR: Packages installation failed" ${1}
  if [ $? -eq 0 ] ; then
    grep "====>> Failed" ${1} >> ${LOUT}
    grep "====>> Skipped" ${1} >> ${LOUT}
    grep "====>> Ignored" ${1} >> ${LOUT}
    return 0
  fi

  ### TODO - Add various error detection as they occur

  # Look for generic error
  grep -q "^ERROR: " ${1}
  if [ $? -eq 0 ] ; then
    # Use the search function to get some context
    ${PROGDIR}/../utils/search -s5 "ERROR: " ${1} >>${LOUT}
    return 0
  fi
}

clean_src_repos()
{
  # Clean out source repos which tend to get cranky on force-push
  cRepos="webui freenas samba ports iocage os"
  for r in $cRepos
  do
    if [ -d "${PROFILE}/_BE/${r}" ] ; then
      rm -rf ${PROFILE}/_BE/${r}
    fi
  done
}

# Parse the Pull Description and bring in any other things marked as depends
check_pr_depends()
{
  if [ -z "$ghprbPullLongDescription" ] ; then return 0; fi

  echo "PRDESC: $ghprbPullLongDescription"

  # Are there DEPENDS listed?
  echo "$ghprbPullLongDescription" | grep -q "DEPENDS:"
  if [ $? -ne 0 ] ; then return 0; fi

  local _deps=`echo $ghprbPullLongDescription | sed -n -e 's/^.*DEPENDS: //p' | cut -d '\' -f 1`
  echo "*** Found PR DEPENDS: $_deps ***"

  for prtgt in $_deps
  do

     # Pull the target PR/Repo
     rtgtg=`echo $prtgt | sed 's|http://||g'`
     rtgtg=`echo $prtgt | sed 's|https://||g'`
     rtgtg=`echo $prtgt | sed 's|www.github.com||g'`
     rtgtg=`echo $prtgt | sed 's|github.com||g'`
     tproject=`echo $prtgt | cut -d '/' -f 1`
     trepo=`echo $prtgt | cut -d '/' -f 2`
     tbranch=`echo $prtgt | cut -d '/' -f 3-`
     tbranch=`echo $tbranch | sed 's|^tree/||g'`

     if [ -d "${PROFILE}/_BE/${trepo}" ] ; then
       rm -rf ${PROFILE}/_BE/${trepo}
     else
       echo "*** Warning, no such git repo currently checked out: $trepo***"
     fi

     echo "*** Cloning DEPENDS repo $tproject/$trepo $tbranch***"
     git clone --depth=1 -b ${tbranch} https://github.com/${tproject}/${trepo} ${PROFILE}/_BE/${trepo} 2>@1 >/tmp/.ghClone.$$
     if [ $? -ne 0 ] ; then
	cat /tmp/.ghClone.$$
	rm /tmp/.ghClone.$$
	echo "**** ERROR: Failed to clone the repo https://github.com/$tproject/$trepo -b $tbranch****"
	exit 1
     fi
     rm /tmp/.ghClone.$$
  done
}

# Allowed settings for BUILDINCREMENTAL
case $BUILDINCREMENTAL in
   ON|on|YES|yes|true|TRUE) BUILDINCREMENTAL="true" ;;
   *) BUILDINCREMENTAL="false" ;;
esac

# Set local location of FreeNAS build
if [ -n "$BUILDTAG" ] ; then
  FNASBDIR="/$BUILDTAG"
else
  FNASBDIR="/freenas"
fi
export FNASBDIR

# Error output log
LOUT="/tmp/fnas-error-debug.txt"
touch ${LOUT}

get_bedir

# Allow these defaults to be overridden
TMPFSWORK="all"
BCONF="/usr/local/etc/poudriere-builders.conf"
if [ -e "$BCONF" ] ; then
  grep -q "^FNBUILDERS=" ${BCONF}
  if [ $? -eq 0 ] ; then
    POUDRIERE_JOBS=$(grep "^FNBUILDERS=" ${BCONF} | cut -d '=' -f 2)
    echo "Setting POUDRIERE_JOBS=$POUDRIERE_JOBS"
    export POUDRIERE_JOBS
  else
    CPUS=$(sysctl -n kern.smp.cpus)
    if [ $CPUS -gt 10 ] ; then
      echo "Setting POUDRIERE_JOBS=10"
      export POUDRIERE_JOBS="10"
    fi
  fi
  grep -q "^TMPFSWORK=" ${BCONF}
  if [ $? -eq 0 ] ; then
    TMPFSWORK=$(grep "^TMPFSWORK=" ${BCONF} | cut -d '=' -f 2)
  fi
else
  # Some tuning for our big build boxes
  CPUS=$(sysctl -n kern.smp.cpus)
  if [ $CPUS -gt 10 ] ; then
    echo "Setting POUDRIERE_JOBS=10"
    export POUDRIERE_JOBS="10"
  fi
fi

# Rotate an old build
if [ -d "${FNASBDIR}" -a "${BUILDINCREMENTAL}" != "true" ] ; then
  echo "Doing fresh build!"
  cd ${FNASBDIR}
  chflags -R 0 ${BEDIR}
  rm -rf ${BEDIR}
fi

# If this is a github pull request builder, check if branch needs to be overridden
if [ -n "$ghprbTargetBranch" ] ; then
  GITFNASBRANCH="$ghprbTargetBranch"
  echo "*** Building GitHub PR, using builder branch: $GITFNASBRANCH ***"
  newTrain="PR-${PRBUILDER}-`echo $ghprbSourceBranch | sed 's|/|-|g'`"
  echo "*** Setting new TRAIN=$newTrain ***"
  BUILDOPTS="$BUILDOPTS TRAIN=$newTrain"
fi

if [ "$BUILDINCREMENTAL" = "true" ] ; then
  echo "Doing incremental build!"
  cd ${FNASBDIR}

  # Nuke old ISO's / builds
  echo "Removing old build ISOs"
  rm -rf ${BEDIR}/release 2>/dev/null
fi

# Figure out the flavor for this test
echo $BUILDTAG | grep -q "truenas"
if [ $? -eq 0 ] ; then
  FLAVOR="TRUENAS"
else
  FLAVOR="FREENAS"
fi

# Add JENKINSBUILDSENV to one specified by the build itself
if [ -n "$JENKINSBUILDSENV" ] ; then<commit_msg>Fix how we parse out the github params<commit_after>#!/usr/bin/env bash

# Where is the pcbsd-build program installed
PROGDIR="`realpath | sed 's|/scripts||g'`" ; export PROGDIR

# Source the config file
. ${PROGDIR}/freenas.cfg

cd ${PROGDIR}/scripts

# Source our functions
. ${PROGDIR}/scripts/functions.sh
. ${PROGDIR}/scripts/functions-tests.sh

# Look through the output log and try to determine the failure
parse_checkout_error()
{
  ### TODO - Add error detection of checkout failures
  echo '' > ${LOUT}
}

# Look through the output log and try to determine the failure
parse_build_error()
{
  echo '' > ${LOUT}
  export TESTSTDERR=${LOUT}

  # Look for some of the common error messages

  # port failed to compile
  grep -q "ERROR: Packages installation failed" ${1}
  if [ $? -eq 0 ] ; then
    grep "====>> Failed" ${1} >> ${LOUT}
    grep "====>> Skipped" ${1} >> ${LOUT}
    grep "====>> Ignored" ${1} >> ${LOUT}
    return 0
  fi

  ### TODO - Add various error detection as they occur

  # Look for generic error
  grep -q "^ERROR: " ${1}
  if [ $? -eq 0 ] ; then
    # Use the search function to get some context
    ${PROGDIR}/../utils/search -s5 "ERROR: " ${1} >>${LOUT}
    return 0
  fi
}

clean_src_repos()
{
  # Clean out source repos which tend to get cranky on force-push
  cRepos="webui freenas samba ports iocage os"
  for r in $cRepos
  do
    if [ -d "${PROFILE}/_BE/${r}" ] ; then
      rm -rf ${PROFILE}/_BE/${r}
    fi
  done
}

# Parse the Pull Description and bring in any other things marked as depends
check_pr_depends()
{
  if [ -z "$ghprbPullLongDescription" ] ; then return 0; fi

  echo "PRDESC: $ghprbPullLongDescription"

  # Are there DEPENDS listed?
  echo "$ghprbPullLongDescription" | grep -q "DEPENDS:"
  if [ $? -ne 0 ] ; then return 0; fi

  local _deps=`echo $ghprbPullLongDescription | sed -n -e 's/^.*DEPENDS: //p' | cut -d '\' -f 1`
  echo "*** Found PR DEPENDS: $_deps ***"

  for prtgt in $_deps
  do

     # Pull the target PR/Repo
     tgt=`echo $prtgt | sed 's|http://||g'`
     tgt=`echo $tgt | sed 's|https://||g'`
     tgt=`echo $tgt | sed 's|www.github.com||g'`
     tgt=`echo $tgt | sed 's|github.com||g'`
     project=`echo $tgt | cut -d '/' -f 1`
     trepo=`echo $tgt | cut -d '/' -f 2`
     tbranch=`echo $tgt | cut -d '/' -f 3-`
     tbranch=`echo $tbranch | sed 's|^tree/||g'`

     if [ -d "${PROFILE}/_BE/${trepo}" ] ; then
       rm -rf ${PROFILE}/_BE/${trepo}
     else
       echo "*** Warning, no such git repo currently checked out: $trepo***"
     fi

     echo "*** Cloning DEPENDS repo https://github.com/$tproject/$trepo $tbranch***"
     git clone --depth=1 -b ${tbranch} https://github.com/${tproject}/${trepo} ${PROFILE}/_BE/${trepo} 2>@1 >/tmp/.ghClone.$$
     if [ $? -ne 0 ] ; then
	cat /tmp/.ghClone.$$
	rm /tmp/.ghClone.$$
	echo "**** ERROR: Failed to clone the repo https://github.com/$tproject/$trepo -b $tbranch****"
	exit 1
     fi
     rm /tmp/.ghClone.$$
  done
}

# Allowed settings for BUILDINCREMENTAL
case $BUILDINCREMENTAL in
   ON|on|YES|yes|true|TRUE) BUILDINCREMENTAL="true" ;;
   *) BUILDINCREMENTAL="false" ;;
esac

# Set local location of FreeNAS build
if [ -n "$BUILDTAG" ] ; then
  FNASBDIR="/$BUILDTAG"
else
  FNASBDIR="/freenas"
fi
export FNASBDIR

# Error output log
LOUT="/tmp/fnas-error-debug.txt"
touch ${LOUT}

get_bedir

# Allow these defaults to be overridden
TMPFSWORK="all"
BCONF="/usr/local/etc/poudriere-builders.conf"
if [ -e "$BCONF" ] ; then
  grep -q "^FNBUILDERS=" ${BCONF}
  if [ $? -eq 0 ] ; then
    POUDRIERE_JOBS=$(grep "^FNBUILDERS=" ${BCONF} | cut -d '=' -f 2)
    echo "Setting POUDRIERE_JOBS=$POUDRIERE_JOBS"
    export POUDRIERE_JOBS
  else
    CPUS=$(sysctl -n kern.smp.cpus)
    if [ $CPUS -gt 10 ] ; then
      echo "Setting POUDRIERE_JOBS=10"
      export POUDRIERE_JOBS="10"
    fi
  fi
  grep -q "^TMPFSWORK=" ${BCONF}
  if [ $? -eq 0 ] ; then
    TMPFSWORK=$(grep "^TMPFSWORK=" ${BCONF} | cut -d '=' -f 2)
  fi
else
  # Some tuning for our big build boxes
  CPUS=$(sysctl -n kern.smp.cpus)
  if [ $CPUS -gt 10 ] ; then
    echo "Setting POUDRIERE_JOBS=10"
    export POUDRIERE_JOBS="10"
  fi
fi

# Rotate an old build
if [ -d "${FNASBDIR}" -a "${BUILDINCREMENTAL}" != "true" ] ; then
  echo "Doing fresh build!"
  cd ${FNASBDIR}
  chflags -R 0 ${BEDIR}
  rm -rf ${BEDIR}
fi

# If this is a github pull request builder, check if branch needs to be overridden
if [ -n "$ghprbTargetBranch" ] ; then
  GITFNASBRANCH="$ghprbTargetBranch"
  echo "*** Building GitHub PR, using builder branch: $GITFNASBRANCH ***"
  newTrain="PR-${PRBUILDER}-`echo $ghprbSourceBranch | sed 's|/|-|g'`"
  echo "*** Setting new TRAIN=$newTrain ***"
  BUILDOPTS="$BUILDOPTS TRAIN=$newTrain"
  rm -rf "${WORKSPACE}/artifacts"
fi

if [ "$BUILDINCREMENTAL" = "true" ] ; then
  echo "Doing incremental build!"
  cd ${FNASBDIR}

  # Nuke old ISO's / builds
  echo "Removing old build ISOs"
  rm -rf ${BEDIR}/release 2>/dev/null
fi

# Figure out the flavor for this test
echo $BUILDTAG | grep -q "truenas"
if [ $? -eq 0 ] ; then
  FLAVOR="TRUENAS"
else
  FLAVOR="FREENAS"
fi

# Add JENKINSBUILDSENV to one specified by the build itself
if [ -n "$JENKINSBUILDSENV" ] ; then
{'fraction_non_alphanumeric': 0.16086155040263897, 'fraction_numerical': 0.008149801106044435, 'mean_word_length': 3.188541243396993, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 41, 'https://': 7, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1336349', 'n_tokens_mistral': 4129, 'n_tokens_neox': 3996, 'n_words': 1153}