<commit_before>#!/bin/sh

# Where is the build program installed
PROGDIR="`realpath | sed 's|/scripts||g'`" ; export PROGDIR

# Source the config file
. ${PROGDIR}/trueos.cfg

cd ${PROGDIR}/scripts

# Source our functions
. ${PROGDIR}/scripts/functions.sh

create_dist_files() {

  # Create the FreeBSD Dist Files
  if [ -n "${DISTDIR}" -a -d "${DISTDIR}" ] ; then
    rm -rf ${DISTDIR}
  fi
  mkdir ${DISTDIR} 2>/dev/null

 # cd to release dir, and clean and make
  cd ${WORLDSRC}/release
  make clean ${SYS_MAKEFLAGS}

  # Create the FTP files
  make ftp NOPORTS=yes ${SYS_MAKEFLAGS}
  if [ $? -ne 0 ] ; then
     echo "Failed running: make ftp NOPORTS=yes ${SYS_MAKEFLAGS}"
     exit 1
  fi
  rc_halt "mv ${WORLDSRC}/release/ftp/* ${DISTDIR}/"

  # Cleanup old .txz files
  cd ${WORLDSRC}/release
  make clean ${SYS_MAKEFLAGS}

  # Dirty dirty hack to get llvm38 into our base packages
  # specifically for buildign in poudriere
  rm -rf ${DISTDIR}/llvm-src 2>/dev/null
  chflags -R noschg ${DISTDIR}/llvm-src 2>/dev/null
  rm -rf ${DISTDIR}/llvm-src 2>/dev/null

  mkdir ${DISTDIR}/llvm-src
  tar xvpf ${DISTDIR}/base.txz -C ${DISTDIR}/llvm-src
  if [ $? -ne 0 ] ; then
     echo "Failed running: tar xvpf base.txz"
     exit 1
  fi

  pkg info -l llvm38 | tail -n +2 | awk '{print $1}' > /tmp/.llvmPlist

  echo "Extracting llvm38 to base"
  tar cvf - -T /tmp/.llvmPlist 2>/dev/null | tar xvpf - -C ${DISTDIR}/llvm-src

  tar cvJf ${DISTDIR}/base.txz -C ${DISTDIR}/llvm-src .
  if [ $? -ne 0 ] ; then
     echo "Failed running: tar cvJf base.txz"
     exit 1
  fi

  # Remove temp dir
  rm -rf ${DISTDIR}/llvm-src 2>/dev/null
  chflags -R noschg ${DISTDIR}/llvm-src 2>/dev/null
  rm -rf ${DISTDIR}/llvm-src 2>/dev/null

  # Re-run make-manifest
  cd ${DISTDIR}
  ${WORLDSRC}/release/scripts/make-manifest.sh *.txz > MANIFEST

  return 0
}

create_base_pkg_files()
{
  cd ${WORLDSRC}

  if [ -n "${PROGDIR}/fbsd-pkg" -a -d "${PROGDIR}/fbsd-pkg" ] ; then
    rm -rf ${PROGDIR}/fbsd-pkg
  fi
  mkdir ${PROGDIR}/fbsd-pkg 2>/dev/null

  # Unset some variables which may be getting in the way
  ODISTDIR="$DISTDIR"
  OWORLDSRC="$WORLDSRC"
  unset DISTDIR WORLDSRC

  # Create the package files now
  make packages ${SYS_MAKEFLAGS}
  if [ $? -ne 0 ] ; then
     env
     echo "Failed running: make packages ${SYS_MAKEFLAGS}"
     exit 1
  fi

  # Move the package files and prep them
  mv /usr/obj/usr/src/repo/*/latest/* ${PROGDIR}/fbsd-pkg/
  if [ $? -ne 0 ] ; then
     echo "Failed moving packages"
     exit 1
  fi

  # This is super ugly, remove it once they properly fix pkg
  # grab all the distrib files
  rc_halt "mkdir ${PROGDIR}/fbsd-distrib"
  cd /usr/src
  make distrib-dirs DESTDIR=${PROGDIR}/fbsd-distrib ${SYS_MAKEFLAGS}
  if [ $? -ne 0 ] ; then
     env
     echo "Failed running: make distrib-dirs ${SYS_MAKEFLAGS}"
     exit 1
  fi
  make distribution DESTDIR=${PROGDIR}/fbsd-distrib ${SYS_MAKEFLAGS}
  if [ $? -ne 0 ] ; then
     env
     echo "Failed running: make distribution ${SYS_MAKEFLAGS}"
     exit 1
  fi

  # Couple of files also missed by pkg base
  rc_halt "tar xvpf ${ODISTDIR}/base.txz -C ${PROGDIR}/fbsd-distrib ./usr/lib/libgcc_eh.a ./usr/lib/libgcc_eh_p.a"
  rc_halt "tar xvpf ${ODISTDIR}/base.txz -C ${PROGDIR}/fbsd-distrib ./usr/share/examples"
  rc_halt "tar xvpf ${ODISTDIR}/base.txz -C ${PROGDIR}/fbsd-distrib ./usr/share/calendar"
  rc_halt "tar xvpf ${ODISTDIR}/base.txz -C ${PROGDIR}/fbsd-distrib ./usr/share/tmac"
  rc_halt "tar xvpf ${ODISTDIR}/base.txz -C ${PROGDIR}/fbsd-distrib ./usr/include"

  # Signing script
  if [ -n "$PKGSIGNCMD" ] ; then
    echo "Signing base packages..."
    rc_halt "cd ${PROGDIR}/fbsd-pkg/"
    rc_halt "pkg repo . signing_command: ${PKGSIGNCMD}"
  fi

  rc_halt "tar cvJf ${PROGDIR}/fbsd-pkg/fbsd-distrib.txz -C ${PROGDIR}/fbsd-distrib ."
  rm -rf ${PROGDIR}/fbsd-distrib

  WORLDSRC="$OWORLDSRC"
  DISTDIR="$ODISTDIR"
  return 0
}<commit_msg>Sign the fbsd-distrib.txz file<commit_after>#!/bin/sh

# Where is the build program installed
PROGDIR="`realpath | sed 's|/scripts||g'`" ; export PROGDIR

# Source the config file
. ${PROGDIR}/trueos.cfg

cd ${PROGDIR}/scripts

# Source our functions
. ${PROGDIR}/scripts/functions.sh

create_dist_files() {

  # Create the FreeBSD Dist Files
  if [ -n "${DISTDIR}" -a -d "${DISTDIR}" ] ; then
    rm -rf ${DISTDIR}
  fi
  mkdir ${DISTDIR} 2>/dev/null

 # cd to release dir, and clean and make
  cd ${WORLDSRC}/release
  make clean ${SYS_MAKEFLAGS}

  # Create the FTP files
  make ftp NOPORTS=yes ${SYS_MAKEFLAGS}
  if [ $? -ne 0 ] ; then
     echo "Failed running: make ftp NOPORTS=yes ${SYS_MAKEFLAGS}"
     exit 1
  fi
  rc_halt "mv ${WORLDSRC}/release/ftp/* ${DISTDIR}/"

  # Cleanup old .txz files
  cd ${WORLDSRC}/release
  make clean ${SYS_MAKEFLAGS}

  # Dirty dirty hack to get llvm38 into our base packages
  # specifically for buildign in poudriere
  rm -rf ${DISTDIR}/llvm-src 2>/dev/null
  chflags -R noschg ${DISTDIR}/llvm-src 2>/dev/null
  rm -rf ${DISTDIR}/llvm-src 2>/dev/null

  mkdir ${DISTDIR}/llvm-src
  tar xvpf ${DISTDIR}/base.txz -C ${DISTDIR}/llvm-src
  if [ $? -ne 0 ] ; then
     echo "Failed running: tar xvpf base.txz"
     exit 1
  fi

  pkg info -l llvm38 | tail -n +2 | awk '{print $1}' > /tmp/.llvmPlist

  echo "Extracting llvm38 to base"
  tar cvf - -T /tmp/.llvmPlist 2>/dev/null | tar xvpf - -C ${DISTDIR}/llvm-src

  tar cvJf ${DISTDIR}/base.txz -C ${DISTDIR}/llvm-src .
  if [ $? -ne 0 ] ; then
     echo "Failed running: tar cvJf base.txz"
     exit 1
  fi

  # Remove temp dir
  rm -rf ${DISTDIR}/llvm-src 2>/dev/null
  chflags -R noschg ${DISTDIR}/llvm-src 2>/dev/null
  rm -rf ${DISTDIR}/llvm-src 2>/dev/null

  # Re-run make-manifest
  cd ${DISTDIR}
  ${WORLDSRC}/release/scripts/make-manifest.sh *.txz > MANIFEST

  return 0
}

create_base_pkg_files()
{
  cd ${WORLDSRC}

  if [ -n "${PROGDIR}/fbsd-pkg" -a -d "${PROGDIR}/fbsd-pkg" ] ; then
    rm -rf ${PROGDIR}/fbsd-pkg
  fi
  mkdir ${PROGDIR}/fbsd-pkg 2>/dev/null

  # Unset some variables which may be getting in the way
  ODISTDIR="$DISTDIR"
  OWORLDSRC="$WORLDSRC"
  unset DISTDIR WORLDSRC

  # Create the package files now
  make packages ${SYS_MAKEFLAGS}
  if [ $? -ne 0 ] ; then
     env
     echo "Failed running: make packages ${SYS_MAKEFLAGS}"
     exit 1
  fi

  # Move the package files and prep them
  mv /usr/obj/usr/src/repo/*/latest/* ${PROGDIR}/fbsd-pkg/
  if [ $? -ne 0 ] ; then
     echo "Failed moving packages"
     exit 1
  fi

  # This is super ugly, remove it once they properly fix pkg
  # grab all the distrib files
  rc_halt "mkdir ${PROGDIR}/fbsd-distrib"
  cd /usr/src
  make distrib-dirs DESTDIR=${PROGDIR}/fbsd-distrib ${SYS_MAKEFLAGS}
  if [ $? -ne 0 ] ; then
     env
     echo "Failed running: make distrib-dirs ${SYS_MAKEFLAGS}"
     exit 1
  fi
  make distribution DESTDIR=${PROGDIR}/fbsd-distrib ${SYS_MAKEFLAGS}
  if [ $? -ne 0 ] ; then
     env
     echo "Failed running: make distribution ${SYS_MAKEFLAGS}"
     exit 1
  fi

  # Couple of files also missed by pkg base
  rc_halt "tar xvpf ${ODISTDIR}/base.txz -C ${PROGDIR}/fbsd-distrib ./usr/lib/libgcc_eh.a ./usr/lib/libgcc_eh_p.a"
  rc_halt "tar xvpf ${ODISTDIR}/base.txz -C ${PROGDIR}/fbsd-distrib ./usr/share/examples"
  rc_halt "tar xvpf ${ODISTDIR}/base.txz -C ${PROGDIR}/fbsd-distrib ./usr/share/calendar"
  rc_halt "tar xvpf ${ODISTDIR}/base.txz -C ${PROGDIR}/fbsd-distrib ./usr/share/tmac"
  rc_halt "tar xvpf ${ODISTDIR}/base.txz -C ${PROGDIR}/fbsd-distrib ./usr/include"

  # Signing script
  if [ -n "$PKGSIGNCMD" ] ; then
    echo "Signing base packages..."
    rc_halt "cd ${PROGDIR}/fbsd-pkg/"
    rc_halt "pkg repo . signing_command: ${PKGSIGNCMD}"
  fi

  rc_halt "tar cvJf ${PROGDIR}/fbsd-pkg/fbsd-distrib.txz -C ${PROGDIR}/fbsd-distrib ."
  rc_halt "openssl dgst -sha1 -sign /etc/ssl/pcbsd-pkgng.key -out ${PROGDIR}/fbsd-distrib.txz.sha1 ${PROGDIR}/fbsd-distrib.txz"
  rm -rf ${PROGDIR}/fbsd-distrib

  WORLDSRC="$OWORLDSRC"
  DISTDIR="$ODISTDIR"
  return 0
}
{'fraction_non_alphanumeric': 0.14607585429966202, 'fraction_numerical': 0.008511703592439605, 'mean_word_length': 3.0150753768844223, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3722933', 'n_tokens_mistral': 3311, 'n_tokens_neox': 3064, 'n_words': 842}