<commit_before>#!/bin/sh
# $1 = ALGO directory, relative to base algo dir
# $2 = COMPUTE env directory, relative to base compute envs dir
# $3 = DATA directory, relative to base data dir

# EG orchestrator.sh 01.java/01.mahout/01.example/ 01.linux/01.centos/01.mahout/ 01.MovieTweetings/datasets/snapshots_10K/

BASEDIR=/home/davide/sources/crowdrec/reference-framework
STAGEDIR=$1
BASEMSG_IN=cmd_in.msg
INF=$BASEDIR/$BASEMSG_IN
OUTF=$BASEDIR/cmd_out.msg

ALGO_DIR=$BASEDIR/../algorithms
DATA_DIR=$BASEDIR/../datasets
COMPUTING_ENV_DIR=$BASEDIR/../computingenvironments
MESSAGING_DIR=$BASEDIR/.messaging

SDIR=./resources/

FNAME=$BASEDIR/output_filename.1

echo "cleaning"
rm -f $FNAME
rm -f $INF
rm -f $OUTF
if [ -f .pid ]; then
	kill -9 `cat .pid`
fi
mkdir -p $MESSAGING_DIR
mkdir -o $MESSAGING_DIR/msg
mkdir -o $MESSAGING_DIR/data


echo "DO: Update git REPOs"
cd $ALGO_DIR
git pull

cd $DATA_DIR
git pull

cd $COMPUTING_ENV_DIR
git pull

# TODO: creating train/test sets

# TODO: separate libraries from core algorithm
echo "DO: starting machine"
cd $COMPUTING_ENV_DIR/$2
SHARED_ALGO=$ALGO_DIR/$1 SHARED_DATA=$DATA_DIR/$3 SHARED_MSG=$MESSAGGING_DIR vagrant up 

echo "STATUS: waiting for machine to be ready"
while [ ! -f $OUTF ] ; 
do
        sleep 2
done<commit_msg>fix for mkdir<commit_after>
#!/bin/sh
# $1 = ALGO directory, relative to base algo dir
# $2 = COMPUTE env directory, relative to base compute envs dir
# $3 = DATA directory, relative to base data dir

# EG orchestrator.sh 01.java/01.mahout/01.example/ 01.linux/01.centos/01.mahout/ 01.MovieTweetings/datasets/snapshots_10K/

BASEDIR=/home/davide/sources/crowdrec/reference-framework
STAGEDIR=$1
BASEMSG_IN=cmd_in.msg
INF=$BASEDIR/$BASEMSG_IN
OUTF=$BASEDIR/cmd_out.msg

ALGO_DIR=$BASEDIR/../algorithms
DATA_DIR=$BASEDIR/../datasets
COMPUTING_ENV_DIR=$BASEDIR/../computingenvironments
MESSAGING_DIR=$BASEDIR/.messaging

SDIR=./resources/

FNAME=$BASEDIR/output_filename.1

echo "cleaning"
rm -f $FNAME
rm -f $INF
rm -f $OUTF
if [ -f .pid ]; then
	kill -9 `cat .pid`
fi
mkdir -p $MESSAGING_DIR
mkdir -p $MESSAGING_DIR/msg
mkdir -p $MESSAGING_DIR/data


echo "DO: Update git REPOs"
cd $ALGO_DIR
git pull

cd $DATA_DIR
git pull

cd $COMPUTING_ENV_DIR
git pull

# TODO: creating train/test sets

# TODO: separate libraries from core algorithm
echo "DO: starting machine"
cd $COMPUTING_ENV_DIR/$2
SHARED_ALGO=$ALGO_DIR/$1 SHARED_DATA=$DATA_DIR/$3 SHARED_MSG=$MESSAGGING_DIR vagrant up 

echo "STATUS: waiting for machine to be ready"
while [ ! -f $OUTF ] ; 
do
        sleep 2
done
{'fraction_non_alphanumeric': 0.1154299175500589, 'fraction_numerical': 0.020416175893207697, 'mean_word_length': 4.539130434782609, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4412735', 'n_tokens_mistral': 1027, 'n_tokens_neox': 992, 'n_words': 252}