<commit_before>#!/bin/bash
###############################################################################################################
#                                                                                                             #
#                                  Cassandra Node Snapshot Launcher for Slurm                                 #
#                                          <NAME> - <EMAIL>                                         #
#                                                                                                             #
#                                        Barcelona Supercomputing Center                                      #
#                                                    .-.--_                                                   #
#                                                  ,´,´.´   `.                                                #
#                                                  | | | BSC |                                                #
#                                                  `.`.`. _ .´                                                #
#                                                    `·`··                                                    #
#                                                                                                             #
###############################################################################################################

C4S_HOME=$HOME/.c4s
CFG_FILE=$C4S_HOME/conf/cassandra4slurm.cfg
source $CFG_FILE
SNAP_NAME=${1}
ROOT_PATH=${2}
CLUSTER=${3}
UNIQ_ID=${4}
DATA_HOME=$ROOT_PATH/cassandra-data
CASSFILE=$C4S_HOME/casslist-"$UNIQ_ID".txt
RINGFILE=$C4S_HOME/ringfile-"$UNIQ_ID".txt
RINGDONE=$C4S_HOME/ringdone-"$UNIQ_ID".txt
HST_IFACE="-ib0" #interface configured in the cassandra.yaml file


casslist=`cat $CASSFILE`

mkdir -p $SNAP_PATH

if [ "x$REPLICA_FACTOR" != "x" ]; then
    if [[ $REPLICA_FACTOR > 1 ]]; then
        echo " [I] Repairing cassandra"
        # The repair is only needed if replication_factor > 1
        for u_host in $casslist
        do
            $CASS_HOME/bin/nodetool -h ${u_host} repair
        done
    fi
fi

first_node=`head -n1 $CASSFILE`

rm -f $RINGFILE $RINGDONE
$CASS_HOME/bin/nodetool -h ${first_node} ring > $RINGFILE
echo "1" > $RINGDONE


for u_host in $casslist
do
    $CASS_HOME/bin/nodetool -h ${u_host} snapshot -t $SNAP_NAME
    $CASS_HOME/bin/nodetool -h ${u_host} drain
done

sacct --delimiter="," -pj ${SLURM_JOB_ID} | grep cass_node | awk -F ',' '{print $1}' | xargs scancel

srun --nodelist=$CASSANDRA_NODELIST --ntasks=$N_NODES --ntasks-per-node=1 --cpus-per-task=$C4S_CASSANDRA_CORES --nodes=$N_NODES $MODULE_PATH/copy_snapshot.sh $SNAP_NAME $ROOT_PATH $CLUSTER $UNIQ_ID
<commit_msg>Add some information messages<commit_after>#!/bin/bash
###############################################################################################################
#                                                                                                             #
#                                  Cassandra Node Snapshot Launcher for Slurm                                 #
#                                          <NAME> - <EMAIL>                                         #
#                                                                                                             #
#                                        Barcelona Supercomputing Center                                      #
#                                                    .-.--_                                                   #
#                                                  ,´,´.´   `.                                                #
#                                                  | | | BSC |                                                #
#                                                  `.`.`. _ .´                                                #
#                                                    `·`··                                                    #
#                                                                                                             #
###############################################################################################################

C4S_HOME=$HOME/.c4s
CFG_FILE=$C4S_HOME/conf/cassandra4slurm.cfg
source $CFG_FILE
SNAP_NAME=${1}
ROOT_PATH=${2}
CLUSTER=${3}
UNIQ_ID=${4}
DATA_HOME=$ROOT_PATH/cassandra-data
CASSFILE=$C4S_HOME/casslist-"$UNIQ_ID".txt
RINGFILE=$C4S_HOME/ringfile-"$UNIQ_ID".txt
RINGDONE=$C4S_HOME/ringdone-"$UNIQ_ID".txt
HST_IFACE="-ib0" #interface configured in the cassandra.yaml file


casslist=`cat $CASSFILE`

echo " [I] Creating Snapshot directory $SNAP_PATH"
mkdir -p $SNAP_PATH

if [ "x$REPLICA_FACTOR" != "x" ]; then
    if [[ $REPLICA_FACTOR > 1 ]]; then
        echo " [I] Repairing cassandra"
        # The repair is only needed if replication_factor > 1
        for u_host in $casslist
        do
            $CASS_HOME/bin/nodetool -h ${u_host} repair
        done
    fi
fi

first_node=`head -n1 $CASSFILE`

echo " [I] Generating RingFile $RINGFILE"
rm -f $RINGFILE $RINGDONE
$CASS_HOME/bin/nodetool -h ${first_node} ring > $RINGFILE
echo "1" > $RINGDONE


echo " [I] Generating Snapshot $SNAP_NAME"
for u_host in $casslist
do
    $CASS_HOME/bin/nodetool -h ${u_host} snapshot -t $SNAP_NAME
    $CASS_HOME/bin/nodetool -h ${u_host} drain
done
echo " [I] Snapshot Done. Copying to GPFS"

sacct --delimiter="," -pj ${SLURM_JOB_ID} | grep cass_node | awk -F ',' '{print $1}' | xargs scancel

srun --nodelist=$CASSANDRA_NODELIST --ntasks=$N_NODES --ntasks-per-node=1 --cpus-per-task=$C4S_CASSANDRA_CORES --nodes=$N_NODES $MODULE_PATH/copy_snapshot.sh $SNAP_NAME $ROOT_PATH $CLUSTER $UNIQ_ID

{'fraction_non_alphanumeric': 0.16727272727272727, 'fraction_numerical': 0.00658008658008658, 'mean_word_length': 1.0988372093023255, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 82, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2252254', 'n_tokens_mistral': 1499, 'n_tokens_neox': 1428, 'n_words': 278}