<commit_before>#!/usr/local/bin/bash
#version 0.2.51

CONFIG="/root/scripts/cloud_backup.conf"
# Read config file
REMOVEOLDINCCOUNT='1'
CRON_MTIME='1'
if [ ! -z "$CONFIG" -a -f "$CONFIG" ];
then
  . $CONFIG
else
  echo "Errors: 1 can't find config file! (${CONFIG})" >&2
  usage
  exit 1
fi

export PATH="/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin"
DUPLY=`which duplicity`


settmp()
{
  mkdir -p /home/tmp >&/dev/null
  export TMPDIR="${ARCHDIR}"
}

setlogs()
{
  mkdir -p "${LOGDIR}" >&/dev/null
  LOCKFILE="${LOGDIR}/backup.lock"
  LOG="${LOGDIR}/cloud_backup.log"
}

lock()
{
  echo "Attempting to acquire lock ${LOCKFILE}" | awk '{system("date \"+%Y-%m-%d %H:%M:%S\"|tr -d \"\\n\"");print " "$0}' >>${LOG} 2>>${LOG}
  if ( set -o noclobber; echo "$$" > "${LOCKFILE}" ) 2> /dev/null; then
      trap 'EXITCODE=$?; echo "Removing lock. Exit code: ${EXITCODE}" >>${LOG} 2>>${LOG}; rm -f "${LOCKFILE}"' 0
      echo "successfully acquired lock." | awk '{system("date \"+%Y-%m-%d %H:%M:%S\"|tr -d \"\\n\"");print " "$0}' >>${LOG} 2>>${LOG}
  else
      echo "lock failed, could not acquire ${LOCKFILE}" | tee -a ${LOG} >&2
      echo "lock held by $(cat ${LOCKFILE})" | tee -a ${LOG} >&2
      exit 2
  fi
}

include_exclude()
{
#building include list from FLIST
  for include in ${FLIST}   
    do
      TMP=" --include="$include
      INCLUDE=$INCLUDE$TMP
  done

#building exclude list from EXLIST  
  for exclude in ${EXLIST}
      do
      TMP=" --exclude "$exclude
      EXCLUDE=$EXCLUDE$TMP
  done
  
#excluding all nfs and nullfs(bind) mounts
  MOUNTS=`mount|grep "nfs\|nullfs\|bind"|grep -v "sunrpc\|nfsd"|awk '{print $3}'`
  for mount in ${MOUNTS}
      do
      TMP=" --exclude "${mount}
      EXCLUDE=$EXCLUDE$TMP
  done
  
#now excluding everything instead of included
    TMP=" --exclude=**"
    INCLUDE=$INCLUDE$TMP
}

include_file()
{

case `uname -s` in
  Linux)
    for i in `/bin/mount|cut -d ' ' -f 1 | grep -v "swap\|devpts\|proc\|sysfs\|tmpfs" | sort | uniq`; do grep $i /etc/fstab | head -n 1 | awk '{print $2}'; done >/root/scripts/cloud_backup_inc.list
  ;;

  FreeBSD)
    for i in `/sbin/mount|cut -d ' ' -f 1 | grep -v "swap\|devpts\|proc\|sysfs\|tmpfs" | sort | uniq`; do grep $i /etc/fstab | head -n 1 | awk '{print $2}'; done >/root/scripts/cloud_backup_inc.list
  ;;

  *)
    echo "Couldn't detect OS"
  ;;
esac

echo "- **" >>/root/scripts/cloud_backup_inc.list
}

exclude_file()
{
  echo "${EXLIST}" > /root/scripts/cloud_backup_exc.list
  
case `uname -s` in
  Linux)
    /bin/mount |grep "nfs\|nullfs\|bind"|grep -v "sunrpc\|nfsd"|awk '{print $3}' >> /root/scripts/cloud_backup_exc.list
  ;;

  FreeBSD)
    /sbin/mount |grep "nfs\|nullfs\|bind"|grep -v "sunrpc\|nfsd"|awk '{print $3}' >> /root/scripts/cloud_backup_exc.list
  ;;

  *)
    echo "Couldn't detect OS"
  ;;
esac

}

usage()
{
  echo " ALL SETTINGS ARE STORED AT cloud_backup.conf "
  echo "CURRENT VARIABLES:"
  echo ""
  echo "  FULLIFOLDER: We will do full backup, if the last full backup was done '$FULLIFOLDER' (days) ago."
  echo "  REMOVEOLDERTHEN: We will clean backups older the '$REMOVEOLDERTHEN' (days)."
  echo "  We will store only 1 incremental chain."
  echo "  VOLSIZE: We will do '${VOLSIZE}' (Mb) archives."<commit_msg>usage updated, removed some variables<commit_after>#!/usr/local/bin/bash
#version 0.2.52

CONFIG="/root/scripts/cloud_backup.conf"
# Read config file
if [ ! -z "$CONFIG" -a -f "$CONFIG" ];
then
  . $CONFIG
else
  echo "Errors: 1 can't find config file! (${CONFIG})" >&2
  usage
  exit 1
fi

export PATH="/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin"
DUPLY=`which duplicity`


settmp()
{
  mkdir -p /home/tmp >&/dev/null
  export TMPDIR="${ARCHDIR}"
}

setlogs()
{
  mkdir -p "${LOGDIR}" >&/dev/null
  LOCKFILE="${LOGDIR}/backup.lock"
  LOG="${LOGDIR}/cloud_backup.log"
}

lock()
{
  echo "Attempting to acquire lock ${LOCKFILE}" | awk '{system("date \"+%Y-%m-%d %H:%M:%S\"|tr -d \"\\n\"");print " "$0}' >>${LOG} 2>>${LOG}
  if ( set -o noclobber; echo "$$" > "${LOCKFILE}" ) 2> /dev/null; then
      trap 'EXITCODE=$?; echo "Removing lock. Exit code: ${EXITCODE}" >>${LOG} 2>>${LOG}; rm -f "${LOCKFILE}"' 0
      echo "successfully acquired lock." | awk '{system("date \"+%Y-%m-%d %H:%M:%S\"|tr -d \"\\n\"");print " "$0}' >>${LOG} 2>>${LOG}
  else
      echo "lock failed, could not acquire ${LOCKFILE}" | tee -a ${LOG} >&2
      echo "lock held by $(cat ${LOCKFILE})" | tee -a ${LOG} >&2
      exit 2
  fi
}

include_exclude()
{
#building include list from FLIST
  for include in ${FLIST}   
    do
      TMP=" --include="$include
      INCLUDE=$INCLUDE$TMP
  done

#building exclude list from EXLIST  
  for exclude in ${EXLIST}
      do
      TMP=" --exclude "$exclude
      EXCLUDE=$EXCLUDE$TMP
  done
  
#excluding all nfs and nullfs(bind) mounts
  MOUNTS=`mount|grep "nfs\|nullfs\|bind"|grep -v "sunrpc\|nfsd"|awk '{print $3}'`
  for mount in ${MOUNTS}
      do
      TMP=" --exclude "${mount}
      EXCLUDE=$EXCLUDE$TMP
  done
  
#now excluding everything instead of included
    TMP=" --exclude=**"
    INCLUDE=$INCLUDE$TMP
}

include_file()
{

case `uname -s` in
  Linux)
    for i in `/bin/mount|cut -d ' ' -f 1 | grep -v "swap\|devpts\|proc\|sysfs\|tmpfs" | sort | uniq`; do grep $i /etc/fstab | head -n 1 | awk '{print $2}'; done >/root/scripts/cloud_backup_inc.list
  ;;

  FreeBSD)
    for i in `/sbin/mount|cut -d ' ' -f 1 | grep -v "swap\|devpts\|proc\|sysfs\|tmpfs" | sort | uniq`; do grep $i /etc/fstab | head -n 1 | awk '{print $2}'; done >/root/scripts/cloud_backup_inc.list
  ;;

  *)
    echo "Couldn't detect OS"
  ;;
esac

echo "- **" >>/root/scripts/cloud_backup_inc.list
}

exclude_file()
{
  echo "${EXLIST}" > /root/scripts/cloud_backup_exc.list
  
case `uname -s` in
  Linux)
    /bin/mount |grep "nfs\|nullfs\|bind"|grep -v "sunrpc\|nfsd"|awk '{print $3}' >> /root/scripts/cloud_backup_exc.list
  ;;

  FreeBSD)
    /sbin/mount |grep "nfs\|nullfs\|bind"|grep -v "sunrpc\|nfsd"|awk '{print $3}' >> /root/scripts/cloud_backup_exc.list
  ;;

  *)
    echo "Couldn't detect OS"
  ;;
esac

}

usage()
{
  echo " ALL SETTINGS ARE STORED AT cloud_backup.conf "
  echo "CURRENT VARIABLES:"
  echo ""
  echo "  FULLIFOLDER: We will do full backup, if the last full backup was done '$FULLIFOLDER' (days) ago."
  echo "  REMOVEOLDERTHEN: We will clean backups older the '$REMOVEOLDERTHEN' (days)."
  echo "  REMOVEOLDINCCOUNT: We will store '${REMOVEOLDINCCOUNT}' incremental chain(s)."
  echo "  VOLSIZE: We will do '${VOLSIZE}' (Mb) archives."
  echo "  CRON_MTIME: Check is configured for cron running every '${CRON_MTIME}' days."
{'fraction_non_alphanumeric': 0.19085006794504, 'fraction_numerical': 0.008304393779254115, 'mean_word_length': 3.2488774855676716, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '302321', 'n_tokens_mistral': 2692, 'n_tokens_neox': 2532, 'n_words': 682}