<commit_before>#!/bin/bash -lxeE

# Setup paths
BASE_DIR=`pwd`
# Path to the host installation
export SLURM_HOST_PREFIX=$1
NODES=$2
# TODO: has to be parameter too
SLURM_SOURCES=/home/artpol/WORK/Mellanox/src/SLURM/pmix/jenkins/slurm-git/slurm/

# Setup path to SLURM source from github. (shuld be passed through env or args).
if [ ! -d $SLURM_SOURCES ]; then
    echo "No slurm sources found. Nothing to check"
    exit 0
fi

HOST_DIR=$BASE_DIR/prepare_host

ROOT_TAR_NAME="root.tar.bz2"

DEV_IMG_DIR=$BASE_DIR/dev_img
DEV_IMG_NAME="dev_img"

NODE_IMG_DIR=$BASE_DIR/node_img
NODE_IMG_NAME="node_img"

BUILD_DIR=$BASE_DIR/compile_img
BUILD_SRC=$BUILD_DIR/src

FINAL_DIR=$BASE_DIR/cluster_img
CLUSTER_IMG_NAME_FILE=`mktemp`
CLUSTER_IMG=`basename $CLUSTER_IMG_NAME_FILE | tr '[:upper:]' '[:lower:]'`

RUNNING_DIR=$BASE_DIR/run_cluster


fix_developer_image()
{
    # NOTE: we need to use <pipeline> || true to 
    # hide error exit code = 1 from grep
    tmp=`docker images | awk '{ print $1 }' | grep $DEV_IMG_NAME || true`
    if [ -z "$tmp" ]; then
        cd $DEV_IMG_DIR
        ./dev_img.sh
        cd $BASE_DIR
    fi
    tmp=`docker images | awk '{ print $1 }' | grep $NODE_IMG_NAME || true`
    if [ -z "$tmp" ]; then
        cd $NODE_IMG_DIR
        ./node_img.sh
        cd $BASE_DIR
    fi


}

build_all()
{
    if [ -f $FINAL_DIR/$ROOT_TAR_NAME ]; then
        #already build root fs
        return
    fi
    mkdir -p $BUILD_SRC
    cp -R $SLURM_SOURCES $BUILD_SRC
    cd $BUILD_DIR
    tmp_file=`mktemp`
    tmp_image=`basename $tmp_file | tr '[:upper:]' '[:lower:]'`
    ./prepare.sh $tmp_image
    rm $tmp_file
    cd $BASE_DIR
}

create_node_image()
{
    mv $BUILD_DIR/$ROOT_TAR_NAME $FINAL_DIR
    cd $FINAL_DIR
    docker build --no-cache=true -t "$CLUSTER_IMG" .
    cd $BASE_DIR
}

prepare_host()
{
    PREFIX_DIR=$1
    cd $HOST_DIR
    ./prepare_host.sh $PREFIX_DIR
    cd $BASE_DIR
}

run_cluster()
{
    cd $RUNNING_DIR
    ./run_cluster.sh $CLUSTER_IMG $NODES
    cd $BASE_DIR
}

#run_cluster()
#{
#    
#}

if [ "$1" = "--prepare-host" ]; then
    prefix_dir=$2
    prepare_host $prefix_dir
    exit 0
fi

# Make sure that we have developer image ready for use.
# Developer image is the machine with full set of packages 
# nessesary to configure and build SLURM, PMIx, libevent and munge.
fix_developer_image

# Create temporal image based on dev_image and use 
# it to build everything
build_all

# Create final cluster/frontend node image
create_node_image

# run cluster
#run_cluster

# Cleanup temp filename
rm $CLUSTER_IMG_NAME_FILE<commit_msg>Make main script more convinient<commit_after>#!/bin/bash -leE

# Setup paths
BASE_DIR=`pwd`

print_usage()
{
    echo "Usage:"
    echo -e "$0 --prepare [force] <SLURM_HOST_PREFIX>"
    echo -e "\tSLURM will be compiled and installed in SLURM_HOST_PREFIX"
    echo -e "\t\"force\" will force replacement of images and SLURM installation"
    echo -e "$0 --test <SLURM_HOST_PREFIX> <SLURM_SOURCES> <NUM_NODES>"
    echo -e "\tSLURM_HOST_PREFIX - path where SLURM host intallation can be found"
    echo -e "\t                    should be the same as for \"--prepare\""
    echo -e "\tSLURM_SOURCES     - path to the cloned repo that should be tested"
    echo -e "\tNUM_NODES         - number of virtual machines to boot"
}

prepare_host()
{
    PREFIX_DIR=$1
    cd $HOST_DIR

    # Sanity check
    if [ -f "$PREFIX_DIR/bin/srun" ] && [ "$FORCE" != 1]; then
        echo "WARNING: SLURM installation found in $PREFIX_DIR"
        echo "Use --force to overwrite it"
        exit 0
    elif [ -f "$PREFIX_DIR/bin/srun" ]; then
        rm --preserve-root -fR $PREFIX_DIR
    fi

    ./prepare_host.sh $PREFIX_DIR
    cd $BASE_DIR
}

prepare_dev_image()
{
    # NOTE: we need to use <pipeline> || true to 
    # hide error exit code = 1 from grep
    STR=`docker images | awk '{ print $1 }' | grep $DEV_IMG_NAME || true`
    if [ -n "$STR" ] && [ "$FORCE" != 1]; then
        echo "WARNING: developer image was found"
        echo "Use --force to remove it"
    else
        if [ -n "$STR" ]; then
            docker rmi -f $DEV_IMG_NAME
        fi
        cd $DEV_IMG_DIR
        ./dev_img.sh
        cd $BASE_DIR
    fi

    STR=`docker images | awk '{ print $1 }' | grep $NODE_IMG_NAME || true`
    if [ -n "$STR" ] && [ "$FORCE" != 1]; then
        echo "WARNING: node image was found"
        echo "Use --force to remove it"
    else
        if [ -n "$STR" ]; then
            docker rmi -f $NODE_IMG_NAME
        fi
        cd $NODE_IMG_DIR
        ./node_img.sh
        cd $BASE_DIR
    fi
}

build_all()
{
    if [ -f $FINAL_DIR/$ROOT_TAR_NAME ]; then
        #already build root fs
        return
    fi
    mkdir -p $BUILD_SRC
    cp -R $SLURM_SOURCES $BUILD_SRC
    cd $BUILD_DIR
    tmp_file=`mktemp`
    tmp_image=`basename $tmp_file | tr '[:upper:]' '[:lower:]'`
    ./prepare.sh $tmp_image
    rm $tmp_file
    cd $BASE_DIR
}

create_node_image()
{
    mv $BUILD_DIR/$ROOT_TAR_NAME $FINAL_DIR
    cd $FINAL_DIR
    docker build --no-cache=true -t "$CLUSTER_IMG" .
    cd $BASE_DIR
}

run_cluster()
{
    cd $RUNNING_DIR
    ./run_cluster.sh $CLUSTER_IMG $NODES
    cd $BASE_DIR
}

case "$1" in
    "--prepare")
        FORCE=0
        if [ "$2" == "--force" ]; then
            FORCE=1
            shift
        fi
        HOST_DIR=$BASE_DIR/prepare_host
        DEV_IMG_DIR=$BASE_DIR/dev_img
        DEV_IMG_NAME="dev_img"
        NODE_IMG_DIR=$BASE_DIR/node_img
        NODE_IMG_NAME="node_img"
        prepare_host $2
        prepare_dev_image
        ;;
    "--test")
        # Enable bash trace
        ROOT_TAR_NAME="root.tar.bz2"
        BUILD_DIR=$BASE_DIR/compile_img
        BUILD_SRC=$BUILD_DIR/src
        FINAL_DIR=$BASE_DIR/cluster_img
        CLUSTER_IMG_NAME_FILE=`mktemp`
        CLUSTER_IMG=`basename $CLUSTER_IMG_NAME_FILE | tr '[:upper:]' '[:lower:]'`
        RUNNING_DIR=$BASE_DIR/run_cluster

        # Path to the host installation
        export SLURM_HOST_PREFIX=$1
        SLURM_SOURCES=$2
        NODES=$3
        if [ ! -f $SLURM_HOST_PREFIX/bin/sbatch ] || \
           [ ! -f $SLURM_HOST_PREFIX/bin/squeue ]; then
            echo "ERROR: No slurm host installation found."
            print_usage
            exit 1
        fi
        if [ ! -d $SLURM_SOURCES ]; then
            echo "ERROR: No slurm sources found. Nothing to check"
            print_usage
            exit 1
        fi

        set -x
        build_all
        create_node_image
        run_cluster

        # Cleanup temp filename
        rm $CLUSTER_IMG_NAME_FILE
        ;;
    *)
        print_usage
        exit 1
        ;;
esac
{'fraction_non_alphanumeric': 0.11044055522027761, 'fraction_numerical': 0.004978877489438744, 'mean_word_length': 2.3701067615658364, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1702864', 'n_tokens_mistral': 2489, 'n_tokens_neox': 2388, 'n_words': 619}