<commit_before>#!/bin/sh
# ------------------------------------------------------------------------------------------------------------------------------------
# Title			: MongoDB Backup
# Description		: This script performs MongoDB full backup using mongodump and Incremental backup using the Tayra tool
# VERSION		: 1.0
# Author		: <NAME>
# Date			: 16.11.2015
# ------------------------------------------------------------------------------------------------------------------------------------

# Backup Directory Structure
# /data/backup
# ├── archive            				- Old backups
# ├── dump						- Current full backup taken with mongodump
# ├── incremental_backup				- Current Incremental backup with Tayra
# └── tayra  						- Tayra program
#     ├── backup.sh					- Tayra	Incremental Backup script
#     ├── restore.sh					- Tayra Restore Script
#     └── timestamp.out					- Tayra stores oplog after the indicated timestamp present in this file.



FULLBACKUP_FREQUENCY=14  # Default no. of days, can be overridden with command line parameter -d
FULL_BACKUP=true
FULLBACKUP_DIR=/data/backup/dump
LATEST_FULLBACKUP=/data/backup/archive/backup_latest.tar.gz
TAYRA_DIR=/data/backup/tayra
CURRENT_DATE=`date +%F`

usage () {
  echo ""
  echo "USAGE: "
  echo "  $0"
  echo "  $0 [-d 14]"
  echo "    -d number of days between full backup ; Default - 14"
  echo ""
  exit 1
}

while getopts "d:" opt; do
  case $opt in
    d)
      if ! echo $OPTARG | grep -q "[A-Za-z]" && [ -n "$OPTARG" ]
      then
        FULLBACKUP_FREQUENCY=$OPTARG
      else
        usage
      fi
    ;;
    *)
      usage
    ;;
  esac
done


# Cleanup old backup processes
OLD_BACKUP_PROCS=$(ps aux | grep '/data/backup'|grep -v grep|awk '{print $2}')
if [ -n "$OLD_BACKUP_PROCS" ]; then
	echo "Terminating old backup processes"
	kill -1 $OLD_BACKUP_PROCS
fi


# Archive backup and clean up tasks
fullbackup_archive() {
	echo "Archiving previous backup"
	tar -P --numeric-owner --preserve-permissions -czf /data/backup/archive/backup_$CURRENT_DATE.tar.gz /data/backup/dump /data/backup/incremental_backup/
	rm -f /data/backup/archive/backup_latest.tar.gz
	rm -rf /data/backup/dump/* /data/backup/incremental_backup/*
	ln -s /data/backup/archive/backup_$CURRENT_DATE.tar.gz /data/backup/archive/backup_latest.tar.gz
}


# Backup Function
runbackup() {

	#run the full backup 
	if [ "$FULL_BACKUP" = true ] ; then
		# Archive previous backup files
		fullbackup_archive
		
		# Get latest timestamp
		echo "Executing a FULL backup"
		LATEST_DB_TIMESTAMP=`mongo local --eval 'db.oplog.rs.find({}, {ts:1}).sort({$natural:-1}).limit(1).forEach(printjson)'|tail -1| awk -F'[(,]' '{print $2}'`
		echo $LATEST_DB_TIMESTAMP
		EXITSTATUS1=`echo $?`
<commit_msg>modified script to take archive<commit_after>#!/bin/sh
# ------------------------------------------------------------------------------------------------------------------------------------
# Title			: MongoDB Backup
# Description		: This script performs MongoDB full backup using mongodump and Incremental backup using the Tayra tool
# VERSION		: 1.0
# Author		: <NAME>
# Date			: 16.11.2015
# ------------------------------------------------------------------------------------------------------------------------------------

# Backup Directory Structure
# /data/backup
# ├── archive            				- Old backups
# ├── dump						- Current full backup taken with mongodump
# ├── incremental_backup				- Current Incremental backup with Tayra
# └── tayra  						- Tayra program
#     ├── backup.sh					- Tayra	Incremental Backup script
#     ├── restore.sh					- Tayra Restore Script
#     └── timestamp.out					- Tayra stores oplog after the indicated timestamp present in this file.



FULLBACKUP_FREQUENCY=14  # Default no. of days, can be overridden with command line parameter -d
FULL_BACKUP=true
FULLBACKUP_DIR=/data/backup/dump
LATEST_ARCHIVE=/data/backup/archive/archive_latest.tar.gz
TAYRA_DIR=/data/backup/tayra
CURRENT_DATE=`date +%F`

usage () {
  echo ""
  echo "USAGE: "
  echo "  $0"
  echo "  $0 [-d 14]"
  echo "    -d number of days between full backup ; Default - 14"
  echo ""
  exit 1
}

while getopts "d:" opt; do
  case $opt in
    d)
      if ! echo $OPTARG | grep -q "[A-Za-z]" && [ -n "$OPTARG" ]
      then
        FULLBACKUP_FREQUENCY=$OPTARG
      else
        usage
      fi
    ;;
    *)
      usage
    ;;
  esac
done


# Cleanup old backup processes
OLD_BACKUP_PROCS=$(ps aux | grep '/data/backup'|grep -v grep|awk '{print $2}')
if [ -n "$OLD_BACKUP_PROCS" ]; then
	echo "Terminating old backup processes"
	kill -1 $OLD_BACKUP_PROCS
fi


# Archive backup and clean up tasks
fullbackup_archive() {
	echo "Archiving previous backup"
	tar -P --numeric-owner --preserve-permissions -czf /data/backup/archive/archive_$CURRENT_DATE.tar.gz /data/backup/dump /data/backup/incremental_backup/
	rm -f /data/backup/archive/archive_latest.tar.gz
	rm -rf /data/backup/dump/* /data/backup/incremental_backup/*
	ln -s /data/backup/archive/archive_$CURRENT_DATE.tar.gz /data/backup/archive/archive_latest.tar.gz
}


# Backup Function
runbackup() {

	#run the full backup 
	if [ "$FULL_BACKUP" = true ] ; then
		# Archive previous backup files
		fullbackup_archive
		
		# Get latest timestamp
		echo "Executing a FULL backup"
		LATEST_DB_TIMESTAMP=`mongo local --eval 'db.oplog.rs.find({}, {ts:1}).sort({$natural:-1}).limit(1).forEach(printjson)'|tail -1| awk -F'[(,]' '{print $2}'`
		echo $LATEST_DB_TIMESTAMP
		EXITSTATUS1=`echo $?`

{'fraction_non_alphanumeric': 0.209047792776359, 'fraction_numerical': 0.009850419554906968, 'mean_word_length': 2.1859384079023823, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2983259', 'n_tokens_mistral': 1900, 'n_tokens_neox': 1741, 'n_words': 476}