<commit_before>#!/bin/bash
LOCK="/home/${USER}/dlrn.lock"
RSYNC_DEST=$(grep rsyncdest /usr/local/share/dlrn/${USER}/projects.ini | awk -F= '{print $2}')
RSYNC_PORT=$(grep rsyncport /usr/local/share/dlrn/${USER}/projects.ini | awk -F= '{print $2}')
set -e

# if any arguments are given, assume console foreground execution
ARG=$1

exec 200>$LOCK
while !  flock -n 200; do
    if [ -n "$ARG" ]
    then
        echo "DLRN ${USER} running, waiting 30 seconds before retrying."
    fi
    sleep 30
done

if [ ! -d /home/${USER}/dlrn-logs ]; then
    mkdir -p /home/${USER}/dlrn-logs
fi

source ~/.venv/bin/activate
if [ -n "$ARG" ]; then
    LOGFILE=/home/${USER}/dlrn-logs/dlrn-purge.console.$(date +%s).log
    ARGUMENTS=""
else
    LOGFILE=/home/${USER}/dlrn-logs/dlrn-purge.$(date +%s).log
    ARGUMENTS="--older-than 60 -y --exclude-dirs /home/${USER}/data/repos/current,/home/${USER}/data/repos/consistent,/home/${USER}/data/repos/current-passed-ci,/home/${USER}/data/repos/current-tripleo,/home/${USER}/data/repos/current-tripleo-rdo,/home/${USER}/data/repos/puppet-passed-ci"
fi
cd ~/dlrn

set +e
echo `date` "Starting DLRN-purge run." >> $LOGFILE
dlrn-purge --config-file /usr/local/share/dlrn/${USER}/projects.ini ${ARGUMENTS} "$@" 2>> $LOGFILE
RET=$?
echo `date` "DLRN-purge run complete." >> $LOGFILE

# If the purge operation was successful, synchronize to the backup node
if [ ${RET} -eq 0 ]; then
    echo `date` "Starting synchronization to backup server." >> $LOGFILE
    if [ -n "${RSYNC_DEST}" ]; then
        rsync -avz --delete-after --exclude="*.htaccess" -e "ssh -p ${RSYNC_PORT} -o StrictHostKeyChecking=no" /home/${USER}/data/repos/ ${RSYNC_DEST} 2>> $LOGFILE
    fi
    echo `date` "Synchronization to backup server completed." >> $LOGFILE
fi

if [ -n "$ARG" ]; then
    echo Arguments: "$@"
    cat $LOGFILE
fi
exit $RET<commit_msg>When purging, only run rsync once a week<commit_after>#!/bin/bash
LOCK="/home/${USER}/dlrn.lock"
RSYNC_DEST=$(grep rsyncdest /usr/local/share/dlrn/${USER}/projects.ini | awk -F= '{print $2}')
RSYNC_PORT=$(grep rsyncport /usr/local/share/dlrn/${USER}/projects.ini | awk -F= '{print $2}')
set -e

# if any arguments are given, assume console foreground execution
ARG=$1

exec 200>$LOCK
while !  flock -n 200; do
    if [ -n "$ARG" ]
    then
        echo "DLRN ${USER} running, waiting 30 seconds before retrying."
    fi
    sleep 30
done

if [ ! -d /home/${USER}/dlrn-logs ]; then
    mkdir -p /home/${USER}/dlrn-logs
fi

source ~/.venv/bin/activate
if [ -n "$ARG" ]; then
    LOGFILE=/home/${USER}/dlrn-logs/dlrn-purge.console.$(date +%s).log
    ARGUMENTS=""
else
    LOGFILE=/home/${USER}/dlrn-logs/dlrn-purge.$(date +%s).log
    ARGUMENTS="--older-than 60 -y --exclude-dirs /home/${USER}/data/repos/current,/home/${USER}/data/repos/consistent,/home/${USER}/data/repos/current-passed-ci,/home/${USER}/data/repos/current-tripleo,/home/${USER}/data/repos/current-tripleo-rdo,/home/${USER}/data/repos/puppet-passed-ci"
fi
cd ~/dlrn

set +e
echo `date` "Starting DLRN-purge run." >> $LOGFILE
dlrn-purge --config-file /usr/local/share/dlrn/${USER}/projects.ini ${ARGUMENTS} "$@" 2>> $LOGFILE
RET=$?
echo `date` "DLRN-purge run complete." >> $LOGFILE

# If the purge operation was successful, synchronize to the backup node
if [ ${RET} -eq 0 ]; then
    # The sync process can be slow, let's do it only once a week, on Sunday
    if [ "$(date +%u)" = "7" ]; then
        echo `date` "Starting synchronization to backup server." >> $LOGFILE
        if [ -n "${RSYNC_DEST}" ]; then
            rsync -avz --delete-after --exclude="*.htaccess" -e "ssh -p ${RSYNC_PORT} -o StrictHostKeyChecking=no" /home/${USER}/data/repos/ ${RSYNC_DEST} 2>> $LOGFILE
        fi
        echo `date` "Synchronization to backup server completed." >> $LOGFILE
    fi
fi

if [ -n "$ARG" ]; then
    echo Arguments: "$@"
    cat $LOGFILE
fi
exit $RET
{'fraction_non_alphanumeric': 0.18725818932164046, 'fraction_numerical': 0.009543461439257158, 'mean_word_length': 3.391845979614949, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3348175', 'n_tokens_mistral': 1482, 'n_tokens_neox': 1448, 'n_words': 355}