<commit_before>#!/bin/bash
# run on adcmon.cern.ch

# adcmon
WORKDIR=/data/jschovan/PandaBrokerageMon/verify_values
PUBDIR=/var/www/html/PandaBrokerageMon/verify_values
# lxplus
WORKDIR=/afs/cern.ch/user/j/jschovan/scratch0/PandaBrokerageMon/verify_values
PUBDIR=${WORKDIR}/pub

LOGDIR=${WORKDIR}/logs
DATESTRING=$(date +%F.%H%M%S)
INDEX=${PUBDIR}/index.html

mkdir -p ${LOGDIR} ${PUBDIR}

cd ${WORKDIR}

echo "Running at ${DATESTRING}."

# get data from DB
/usr/bin/time python verifyDB.py ${WORKDIR} 2>&1 | tee ${LOGDIR}/log.verifyDB.${DATESTRING}

# process data
/usr/bin/time python verify_values.py ${WORKDIR} 2>&1 | tee ${LOGDIR}/log.verify_values.${DATESTRING}

# pack data
cd ${WORKDIR}
bzip2 DAILYLOGV2.py
bzip2 DAILYLOG.py
bzip2 LASTUPDATED.py
bzip2 LASTUPDATEDV2.py
bzip2 MONTHLYLOG.py


# publish data
cp ${WORKDIR}/DAILYLOGV2.py.bz2 ${PUBDIR}/DAILYLOGV2.py.bz2
cp ${LOGDIR}/log.verify_values.${DATESTRING} ${PUBDIR}/log.verify_values.txt

# bzip files
bzip2 ${LOGDIR}/log.verifyDB.${DATESTRING} ${LOGDIR}/log.verify_values.${DATESTRING}

# create index
cd ${PUBDIR}
echo -n >${INDEX}
for file in $(ls . | grep -v "index.html"); 
do
    lsl=$(ls -l $file)
    html="<div>$(echo $lsl | sed -e "s#$file#<a href=\"./$file\">$file</a>#g" | awk '{printf $5 " " $6 " " $7 " " $8 " " $9 " " $10;}')</div>"
    echo $html >> ${INDEX}
done


<commit_msg>verify_values -- migrate to atlas-adcmon<commit_after>#!/bin/bash
# run on adcmon.cern.ch

MYPYTHON=/usr/bin/python2.5

### adcmon
WORKDIR=/data/jschovan/PandaBrokerageMon/verify_values
#PUBDIR=/var/www/html/PandaBrokerageMon/verify_values
PUBDIR=/data/adcmon-preproduction/PandaBrokerageMon/verify_values.test

### lxplus
#WORKDIR=/afs/cern.ch/user/j/jschovan/scratch0/PandaBrokerageMon/verify_values
#PUBDIR=${WORKDIR}/pub

LOGDIR=${WORKDIR}/logs
DATESTRING=$(date +%F.%H%M%S)
INDEX=${PUBDIR}/index.html

mkdir -p ${LOGDIR} ${PUBDIR}

cd ${WORKDIR}

echo "Running at ${DATESTRING}."

# get data from DB
/usr/bin/time ${MYPYTHON} verifyDB.py ${WORKDIR} 2>&1 | tee ${LOGDIR}/log.verifyDB.${DATESTRING}

# process data
/usr/bin/time ${MYPYTHON} verify_values.py ${WORKDIR} 2>&1 | tee ${LOGDIR}/log.verify_values.${DATESTRING}

# pack data
cd ${WORKDIR}
rm DAILYLOGV2.py.bz2 LASTUPDATEDV2.py.bz2

bzip2 DAILYLOGV2.py
#bzip2 DAILYLOG.py
#bzip2 LASTUPDATED.py
bzip2 LASTUPDATEDV2.py
#bzip2 MONTHLYLOG.py


# publish data
cp ${WORKDIR}/DAILYLOGV2.py.bz2 ${PUBDIR}/DAILYLOGV2.py.bz2
cp ${LOGDIR}/log.verify_values.${DATESTRING} ${PUBDIR}/log.verify_values.txt

# bzip files
bzip2 ${LOGDIR}/log.verifyDB.${DATESTRING} ${LOGDIR}/log.verify_values.${DATESTRING}

# create index
cd ${PUBDIR}
echo -n >${INDEX}
for file in $(ls . | grep -v "index.html"); 
do
    lsl=$(ls -l $file)
    html="<div>$(echo $lsl | sed -e "s#$file#<a href=\"./$file\">$file</a>#g" | awk '{printf $5 " " $6 " " $7 " " $8 " " $9 " " $10;}')</div>"
    echo $html >> ${INDEX}
done



{'fraction_non_alphanumeric': 0.18967297762478486, 'fraction_numerical': 0.01858864027538726, 'mean_word_length': 4.743083003952569, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '960271', 'n_tokens_mistral': 1275, 'n_tokens_neox': 1226, 'n_words': 222}