<commit_before>#!/bin/bash

# Weekly cron job to backup to IRODS location
# Backed-up:
#    MySQL databases:
#       coge
#       wikidb
#       cogelinks
#    Data directories:
#       Wiki web directory
#       SVN code repo
#       CoGe data
# Created: 7/11/12 by mdb

echo `date` "Backup started"

ICMD=/usr/local/bin
DAYS_UNTIL_DELETE=21
VERSION=$(date '+%Y%m%d')
LOCAL=/storage/coge/backup
REMOTE=backup
CONFIG=/opt/apache2/coge/web/coge.conf

#
# Dump databases and copy to remote IRODS location
#
DBPASS=$(grep -P "DBPASS" $CONFIG | cut -d ' ' -f3)
LOCAL_MYSQL=$LOCAL/mysql_$VERSION
mkdir -p $LOCAL_MYSQL
echo `date` "Dumping MySQL databases"
mysqldump -u root -p$DBPASS wikidb -c | gzip -9 > $LOCAL_MYSQL/wikidb.sql.gz
mysqlhotcopy -u root -p $DBPASS cogelinks $LOCAL_MYSQL
mysqlhotcopy --port=3307 -u root -p $DBPASS coge $LOCAL_MYSQL
echo `date` "Pushing MySQL databases to IRODS"
$ICMD/iput -bfr $LOCAL_MYSQL $REMOTE

#
# Remove old database backups
#
echo `date` "Deleting old backups (local & remote)"
LOCAL_DELETIONS=`find $LOCAL/mysql_* -maxdepth 1 -type d -mtime +$DAYS_UNTIL_DELETE`
if [ -n "$LOCAL_DELETIONS" ];
then
   echo delete local $LOCAL_DELETIONS
   rm -rf $LOCAL_DELETIONS
fi
for d in `$ICMD/ils backup | grep 'mysql_' | sed 's/.*\(mysql_.*\)/\1/'`
do
   if [ ! -d $LOCAL/$d ];
   then
      echo delete IRODS backup/$d
      $ICMD/irm -r backup/$d
   fi
done

#
# Sync data directories with remote IRODS location
#
echo `date` "Syncing data directories with IRODS"
$ICMD/icd
$ICMD/irsync -rs /opt/apache2/cogepedia i:$REMOTE/cogepedia
$ICMD/irsync -rs /opt/apache2/r i:$REMOTE/r
$ICMD/irsync -rs /storage/coge/data/genomic_sequence/ i:$REMOTE/genomic_sequence
$ICMD/irsync -rs /storage/coge/data/experiments/ i:$REMOTE/experiments

#
# Sync JEX database with remote IRODS location
#
echo `date` "Syncing JEX db with IRODS"
$ICMD/icd
$ICMD/irsync -rs /opt/Yerba/workflows.db i:$REMOTE/jex

#
# Sync /etc with remote IRODS location
#
echo `date` "Syncing /etc with IRODS"
$ICMD/icd
$ICMD/irsync -rs /etc i:$REMOTE/etc

echo `date` "Backup completed"
<commit_msg>Handle all variations of whitespace in pw extraction<commit_after>#!/bin/bash

# Weekly cron job to backup to IRODS location
# Backed-up:
#    MySQL databases:
#       coge
#       wikidb
#       cogelinks
#    Data directories:
#       Wiki web directory
#       SVN code repo
#       CoGe data
# Created: 7/11/12 by mdb

echo `date` "Backup started"

ICMD=/usr/local/bin
DAYS_UNTIL_DELETE=21
VERSION=$(date '+%Y%m%d')
LOCAL=/storage/coge/backup
REMOTE=backup
CONFIG=/opt/apache2/coge/web/coge.conf

#
# Dump databases and copy to remote IRODS location
#
DBPASS=$(grep -P "DBPASS" $CONFIG | awk '{print $2}')
LOCAL_MYSQL=$LOCAL/mysql_$VERSION
mkdir -p $LOCAL_MYSQL
echo `date` "Dumping MySQL databases"
mysqldump -u root -p$DBPASS wikidb -c | gzip -9 > $LOCAL_MYSQL/wikidb.sql.gz
mysqlhotcopy -u root -p $DBPASS cogelinks $LOCAL_MYSQL
mysqlhotcopy --port=3307 -u root -p $DBPASS coge $LOCAL_MYSQL
echo `date` "Pushing MySQL databases to IRODS"
$ICMD/iput -bfr $LOCAL_MYSQL $REMOTE

#
# Remove old database backups
#
echo `date` "Deleting old backups (local & remote)"
LOCAL_DELETIONS=`find $LOCAL/mysql_* -maxdepth 1 -type d -mtime +$DAYS_UNTIL_DELETE`
if [ -n "$LOCAL_DELETIONS" ];
then
   echo delete local $LOCAL_DELETIONS
   rm -rf $LOCAL_DELETIONS
fi
for d in `$ICMD/ils backup | grep 'mysql_' | sed 's/.*\(mysql_.*\)/\1/'`
do
   if [ ! -d $LOCAL/$d ];
   then
      echo delete IRODS backup/$d
      $ICMD/irm -r backup/$d
   fi
done

#
# Sync data directories with remote IRODS location
#
echo `date` "Syncing data directories with IRODS"
$ICMD/icd
$ICMD/irsync -rs /opt/apache2/cogepedia i:$REMOTE/cogepedia
$ICMD/irsync -rs /opt/apache2/r i:$REMOTE/r
$ICMD/irsync -rs /storage/coge/data/genomic_sequence/ i:$REMOTE/genomic_sequence
$ICMD/irsync -rs /storage/coge/data/experiments/ i:$REMOTE/experiments

#
# Sync JEX database with remote IRODS location
#
echo `date` "Syncing JEX db with IRODS"
$ICMD/icd
$ICMD/irsync -rs /opt/Yerba/workflows.db i:$REMOTE/jex

#
# Sync /etc with remote IRODS location
#
echo `date` "Syncing /etc with IRODS"
$ICMD/icd
$ICMD/irsync -rs /etc i:$REMOTE/etc

echo `date` "Backup completed"

{'fraction_non_alphanumeric': 0.1287034842379711, 'fraction_numerical': 0.008532827684285376, 'mean_word_length': 3.741573033707865, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5142940', 'n_tokens_mistral': 1759, 'n_tokens_neox': 1669, 'n_words': 455}