<commit_before>#!/bin/sh
# -*- tab-width:4;indent-tabs-mode:nil -*-
# ex: ts=4 sw=4 et

# Identify the script name
SCRIPT=`basename "$0"`

RUNNER_SCRIPT_DIR=$(cd ${0%/*} && pwd)
RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}
NODE_TYPE=$(basename "$RUNNER_BASE_DIR")

# Load "GLOBAL_CONFIG" option from here (when it's "yes", it means that environment
# file and config file will be picked from /etc/leofs/$NODE_TYPE instead)
[ -f /etc/leofs/leofs.conf ] && . /etc/leofs/leofs.conf
GLOBAL_CONFIG="$(echo $GLOBAL_CONFIG | tr '[A-Z]' '[a-z]')"


# Check if we need to pick alternate set of config files. Compatible only with global config.
if [ "$NODE_EXTRA_NAME" != "" ]
then
    CONFIG_DIR="${NODE_TYPE}_${NODE_EXTRA_NAME}"
else
    CONFIG_DIR="$NODE_TYPE"
    # Define the variable, even if it wasn't before. We rely on it when parsing "ps" output for "stop" command
    NODE_EXTRA_NAME=
    export NODE_EXTRA_NAME
fi


# Source environment script to allow redefining all other variables
if [ "$GLOBAL_CONFIG" = "yes" ]
then
    CONFIG_PATH=/etc/leofs/$CONFIG_DIR/$SCRIPT.environment
else
    CONFIG_PATH=$RUNNER_BASE_DIR/etc/$SCRIPT.environment
fi

[ -f $CONFIG_PATH ] && . $CONFIG_PATH

if [ "$GLOBAL_CONFIG" = "yes" ]
then
    RUNNER_ETC_DIR=${RUNNER_ETC_DIR:-/etc/leofs/$CONFIG_DIR}
else
    RUNNER_ETC_DIR=${RUNNER_ETC_DIR:-$RUNNER_BASE_DIR/etc}
fi

RUNNER_SCHEMA_DIR=${RUNNER_SCHEMA_DIR:-$RUNNER_BASE_DIR/etc}
RUNNER_LOG_DIR=${RUNNER_LOG_DIR:-$RUNNER_BASE_DIR/log}
# Note the trailing slash on $PIPE_DIR/
PIPE_DIR=/tmp/$RUNNER_BASE_DIR/
RUNNER_USER=${RUNNER_USER:-leofs}

# Make sure this script is running as the appropriate user
if [ ! -z "$RUNNER_USER" ] && [ `whoami` != "$RUNNER_USER" ]; then
    exec sudo -H -u $RUNNER_USER "$RUNNER_SCRIPT_DIR/$SCRIPT" "$@"
fi

# Make sure CWD is set to runner base dir
cd $RUNNER_BASE_DIR

# Make sure log directory exists
mkdir -p $RUNNER_LOG_DIR

help () {
    echo "Usage: $SCRIPT [-type leo_manager|leo_gateway|leo_storage] {start|stop|restart|foreground_start|reboot|ping|console|console_clean|attach|remote_console}"
    echo "Script type is picked from its name, but can be overriden with -type option"
    echo "leo_manager additionally supports commands {dump-mnesia-data|load-mnesia-data}"
}

case "$1" in
    -type)
        case "$2" in
            leo_manager|leo_gateway|leo_storage)
                NODE_TYPE=$2
                shift 2
                ;;
            *)
                help
                exit 1
                ;;
        esac
        ;;
    *)
        case "$SCRIPT" in
            leo_manager|leo_gateway|leo_storage)
                NODE_TYPE=$SCRIPT
                ;;
            *)
                echo "Script name (currently: $SCRIPT) should be one of: leo_manager, leo_gateway, leo_manager"
                echo "Or '-type leo_manger|leo_gateway|leo_storage' has to be supplied as the first option"
                exit 1
                ;;
        esac
        ;;
esac

case "$NODE_TYPE" in
    leo_storage)
        ERLEXEC_MODE=minimal
        ;;
    leo_gateway|leo_manager)
        ERLEXEC_MODE=embedded
        ;;
esac


# Parse out release and erts info
START_ERL=`cat $RUNNER_BASE_DIR/releases/start_erl.data`
ERTS_VSN=${START_ERL% *}
APP_VSN=${START_ERL#* }

# Use releases/VSN/vm.args if it exists otherwise use etc/vm.args
# if [ -e "$RUNNER_BASE_DIR/releases/$APP_VSN/vm.args" ]; then
#     VMARGS_PATH="$RUNNER_BASE_DIR/releases/$APP_VSN/vm.args"
# else
#     VMARGS_PATH="$RUNNER_ETC_DIR/vm.args"
# fi
VMARGS_PATH="$RUNNER_ETC_DIR/vm.args"

# Use releases/VSN/sys.config if it exists otherwise use etc/app.config
# if [ -e "$RUNNER_BASE_DIR/releases/$APP_VSN/sys.config" ]; then
#     CONFIG_PATH="$RUNNER_BASE_DIR/releases/$APP_VSN/sys.config"
# else
#     CONFIG_PATH="$RUNNER_ETC_DIR/app.config"
# fi
CONFIG_PATH="$RUNNER_ETC_DIR/app.config"


#
# Generate conf files - [app.config, vm.args]
#
gen_config() {
    rm -f $RUNNER_ETC_DIR/app.*.config
    rm -f $RUNNER_ETC_DIR/vm.*.args
    ERTS_PATH=$RUNNER_BASE_DIR/erts-$ERTS_VSN/bin
    NODETOOL_LITE="$ERTS_PATH/escript $ERTS_PATH/nodetool"
    # Check for presence of extra config files and process them in alphabetical order, if they exist
    ADDITIONAL_CONFIG_CMD=""
    if [ -d $RUNNER_ETC_DIR/$NODE_TYPE.d ] && ls -A $RUNNER_ETC_DIR/$NODE_TYPE.d/*.conf > /dev/null 2>&1
    then
        for config_name in $RUNNER_ETC_DIR/$NODE_TYPE.d/*.conf
        do
            ADDITIONAL_CONFIG_CMD="$ADDITIONAL_CONFIG_CMD -c $config_name"
        done
    fi
    RES_CUTTLEFISH=`PATH=$ERTS_PATH:$PATH $RUNNER_BASE_DIR/bin/cuttlefish -i $RUNNER_SCHEMA_DIR/$NODE_TYPE.schema -c $RUNNER_ETC_DIR/$NODE_TYPE.conf $ADDITIONAL_CONFIG_CMD -d $RUNNER_ETC_DIR/`

    APP_CONFIG=`find $RUNNER_ETC_DIR/ -type f | grep app.*[0-9].config`
    mv $APP_CONFIG $RUNNER_ETC_DIR/app.config

    VM_ARGS=`find $RUNNER_ETC_DIR/ -type f | grep vm.*[0-9].args`
    mv $VM_ARGS $RUNNER_ETC_DIR/vm.args

    # Sanity check the app.config file
    RES=`$NODETOOL_LITE chkconfig $RUNNER_ETC_DIR/app.config`
    if [ "$RES" != "ok" ]; then
        echo "Error reading $RUNNER_ETC_DIR/app.config"
        echo $RES
        exit 1
    fi
}
case "$1" in
    start|console|foreground_start)
        gen_config
        ;;
    *)
        ;;
esac


# Extract the target node name from node.args
NAME_ARG=`egrep '^-s?name' $VMARGS_PATH`
if [ -z "$NAME_ARG" ]; then
    echo "vm.args needs to have either -name or -sname parameter."
    exit 1
fi

# Extract the name type and name from the NAME_ARG for REMSH
REMSH_TYPE=`echo $NAME_ARG | awk '{print $1}'`
REMSH_NAME=`echo $NAME_ARG | awk '{print $2}'`

# Note the `date +%s`, used to allow multiple remsh to the same node transparently
REMSH_NAME_ARG="$REMSH_TYPE remsh`date +%s`@`echo $REMSH_NAME | awk -F@ '{print $2}'`"
REMSH_REMSH_ARG="-remsh $REMSH_NAME"

# Extract the target cookie
COOKIE_ARG=`grep '^-setcookie' $VMARGS_PATH`
if [ -z "$COOKIE_ARG" ]; then
    echo "vm.args needs to have a -setcookie parameter."
    exit 1
fi

# Add ERTS bin dir to our path
ERTS_PATH=$RUNNER_BASE_DIR/erts-$ERTS_VSN/bin

# Setup command to control the node
NODETOOL="$ERTS_PATH/escript $ERTS_PATH/nodetool $NAME_ARG $COOKIE_ARG"

# Setup remote shell command to control node
REMSH="$ERTS_PATH/erl $REMSH_NAME_ARG $REMSH_REMSH_ARG $COOKIE_ARG"

case "$1" in
    dump-mnesia-data|load-mnesia-data)
        if [ "$NODE_TYPE" != leo_manager ]; then
            echo "Only leo_manager supports {dump-mnesia-data|load-mnesia-data} commands"
            exit 1
        fi
    ;;  # actual implementation follows
esac


# Check the first argument for instructions
case "$1" in
    start)
        # Make sure there is not already a node running
        RES=`$NODETOOL ping`
        if [ "$RES" = "pong" ]; then
            echo "Node is already running!"
            exit 1
        fi

        HEART_COMMAND="$RUNNER_BASE_DIR/bin/$SCRIPT start"
        export HEART_COMMAND

        # Uncomment the below two lines ONLY if you face leo_(manager|storage|gateway) restarted frequently
        # in order to dig down further for identifying the root cause.
        # ERL_CRASH_DUMP_SECONDS=-1 means setting the environment variable to a negative value
        # does not reboot the runtime system until the crash dump file is completely written.
        ## ERL_CRASH_DUMP_SECONDS=-1
        ## export ERL_CRASH_DUMP_SECONDS
        mkdir -p $PIPE_DIR
        shift # remove $1
        $ERTS_PATH/run_erl -daemon $PIPE_DIR $RUNNER_LOG_DIR "exec $RUNNER_BASE_DIR/bin/$SCRIPT console $@" 2>&1
        ;;

    stop)
        # Wait for the node to completely stop...
        case `uname -s` in
            Linux)
                # Only look for processes launched with the same NODE_EXTRA_NAME env variable
                # PID COMMAND+ENVIRONMENT
                PID=`ps axeww o pid,args|\
                    grep "NODE_EXTRA_NAME=$NODE_EXTRA_NAME " |\
                    grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $1}'`
                ;;
            Darwin|FreeBSD|DragonFly|NetBSD|OpenBSD)
                # PID COMMAND
                PID=`ps ax -o pid= -o command=|\
                    grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $1}'`
                ;;
            SunOS)
                # PID COMMAND
                PID=`ps -ef -o pid= -o args=|\
                    grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $1}'`
                ;;
            CYGWIN*)
                # UID PID PPID TTY STIME COMMAND
                PID=`ps -efW|grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $2}'`
                ;;
        esac
        $NODETOOL stop<commit_msg>Add a comment about stopping nodes launched under older version<commit_after>#!/bin/sh
# -*- tab-width:4;indent-tabs-mode:nil -*-
# ex: ts=4 sw=4 et

# Identify the script name
SCRIPT=`basename "$0"`

RUNNER_SCRIPT_DIR=$(cd ${0%/*} && pwd)
RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}
NODE_TYPE=$(basename "$RUNNER_BASE_DIR")

# Load "GLOBAL_CONFIG" option from here (when it's "yes", it means that environment
# file and config file will be picked from /etc/leofs/$NODE_TYPE instead)
[ -f /etc/leofs/leofs.conf ] && . /etc/leofs/leofs.conf
GLOBAL_CONFIG="$(echo $GLOBAL_CONFIG | tr '[A-Z]' '[a-z]')"


# Check if we need to pick alternate set of config files. Compatible only with global config.
if [ "$NODE_EXTRA_NAME" != "" ]
then
    CONFIG_DIR="${NODE_TYPE}_${NODE_EXTRA_NAME}"
else
    CONFIG_DIR="$NODE_TYPE"
    # Define the variable, even if it wasn't before. We rely on it when parsing "ps" output for "stop" command
    NODE_EXTRA_NAME=
    export NODE_EXTRA_NAME
fi


# Source environment script to allow redefining all other variables
if [ "$GLOBAL_CONFIG" = "yes" ]
then
    CONFIG_PATH=/etc/leofs/$CONFIG_DIR/$SCRIPT.environment
else
    CONFIG_PATH=$RUNNER_BASE_DIR/etc/$SCRIPT.environment
fi

[ -f $CONFIG_PATH ] && . $CONFIG_PATH

if [ "$GLOBAL_CONFIG" = "yes" ]
then
    RUNNER_ETC_DIR=${RUNNER_ETC_DIR:-/etc/leofs/$CONFIG_DIR}
else
    RUNNER_ETC_DIR=${RUNNER_ETC_DIR:-$RUNNER_BASE_DIR/etc}
fi

RUNNER_SCHEMA_DIR=${RUNNER_SCHEMA_DIR:-$RUNNER_BASE_DIR/etc}
RUNNER_LOG_DIR=${RUNNER_LOG_DIR:-$RUNNER_BASE_DIR/log}
# Note the trailing slash on $PIPE_DIR/
PIPE_DIR=/tmp/$RUNNER_BASE_DIR/
RUNNER_USER=${RUNNER_USER:-leofs}

# Make sure this script is running as the appropriate user
if [ ! -z "$RUNNER_USER" ] && [ `whoami` != "$RUNNER_USER" ]; then
    exec sudo -H -u $RUNNER_USER "$RUNNER_SCRIPT_DIR/$SCRIPT" "$@"
fi

# Make sure CWD is set to runner base dir
cd $RUNNER_BASE_DIR

# Make sure log directory exists
mkdir -p $RUNNER_LOG_DIR

help () {
    echo "Usage: $SCRIPT [-type leo_manager|leo_gateway|leo_storage] {start|stop|restart|foreground_start|reboot|ping|console|console_clean|attach|remote_console}"
    echo "Script type is picked from its name, but can be overriden with -type option"
    echo "leo_manager additionally supports commands {dump-mnesia-data|load-mnesia-data}"
}

case "$1" in
    -type)
        case "$2" in
            leo_manager|leo_gateway|leo_storage)
                NODE_TYPE=$2
                shift 2
                ;;
            *)
                help
                exit 1
                ;;
        esac
        ;;
    *)
        case "$SCRIPT" in
            leo_manager|leo_gateway|leo_storage)
                NODE_TYPE=$SCRIPT
                ;;
            *)
                echo "Script name (currently: $SCRIPT) should be one of: leo_manager, leo_gateway, leo_manager"
                echo "Or '-type leo_manger|leo_gateway|leo_storage' has to be supplied as the first option"
                exit 1
                ;;
        esac
        ;;
esac

case "$NODE_TYPE" in
    leo_storage)
        ERLEXEC_MODE=minimal
        ;;
    leo_gateway|leo_manager)
        ERLEXEC_MODE=embedded
        ;;
esac


# Parse out release and erts info
START_ERL=`cat $RUNNER_BASE_DIR/releases/start_erl.data`
ERTS_VSN=${START_ERL% *}
APP_VSN=${START_ERL#* }

# Use releases/VSN/vm.args if it exists otherwise use etc/vm.args
# if [ -e "$RUNNER_BASE_DIR/releases/$APP_VSN/vm.args" ]; then
#     VMARGS_PATH="$RUNNER_BASE_DIR/releases/$APP_VSN/vm.args"
# else
#     VMARGS_PATH="$RUNNER_ETC_DIR/vm.args"
# fi
VMARGS_PATH="$RUNNER_ETC_DIR/vm.args"

# Use releases/VSN/sys.config if it exists otherwise use etc/app.config
# if [ -e "$RUNNER_BASE_DIR/releases/$APP_VSN/sys.config" ]; then
#     CONFIG_PATH="$RUNNER_BASE_DIR/releases/$APP_VSN/sys.config"
# else
#     CONFIG_PATH="$RUNNER_ETC_DIR/app.config"
# fi
CONFIG_PATH="$RUNNER_ETC_DIR/app.config"


#
# Generate conf files - [app.config, vm.args]
#
gen_config() {
    rm -f $RUNNER_ETC_DIR/app.*.config
    rm -f $RUNNER_ETC_DIR/vm.*.args
    ERTS_PATH=$RUNNER_BASE_DIR/erts-$ERTS_VSN/bin
    NODETOOL_LITE="$ERTS_PATH/escript $ERTS_PATH/nodetool"
    # Check for presence of extra config files and process them in alphabetical order, if they exist
    ADDITIONAL_CONFIG_CMD=""
    if [ -d $RUNNER_ETC_DIR/$NODE_TYPE.d ] && ls -A $RUNNER_ETC_DIR/$NODE_TYPE.d/*.conf > /dev/null 2>&1
    then
        for config_name in $RUNNER_ETC_DIR/$NODE_TYPE.d/*.conf
        do
            ADDITIONAL_CONFIG_CMD="$ADDITIONAL_CONFIG_CMD -c $config_name"
        done
    fi
    RES_CUTTLEFISH=`PATH=$ERTS_PATH:$PATH $RUNNER_BASE_DIR/bin/cuttlefish -i $RUNNER_SCHEMA_DIR/$NODE_TYPE.schema -c $RUNNER_ETC_DIR/$NODE_TYPE.conf $ADDITIONAL_CONFIG_CMD -d $RUNNER_ETC_DIR/`

    APP_CONFIG=`find $RUNNER_ETC_DIR/ -type f | grep app.*[0-9].config`
    mv $APP_CONFIG $RUNNER_ETC_DIR/app.config

    VM_ARGS=`find $RUNNER_ETC_DIR/ -type f | grep vm.*[0-9].args`
    mv $VM_ARGS $RUNNER_ETC_DIR/vm.args

    # Sanity check the app.config file
    RES=`$NODETOOL_LITE chkconfig $RUNNER_ETC_DIR/app.config`
    if [ "$RES" != "ok" ]; then
        echo "Error reading $RUNNER_ETC_DIR/app.config"
        echo $RES
        exit 1
    fi
}
case "$1" in
    start|console|foreground_start)
        gen_config
        ;;
    *)
        ;;
esac


# Extract the target node name from node.args
NAME_ARG=`egrep '^-s?name' $VMARGS_PATH`
if [ -z "$NAME_ARG" ]; then
    echo "vm.args needs to have either -name or -sname parameter."
    exit 1
fi

# Extract the name type and name from the NAME_ARG for REMSH
REMSH_TYPE=`echo $NAME_ARG | awk '{print $1}'`
REMSH_NAME=`echo $NAME_ARG | awk '{print $2}'`

# Note the `date +%s`, used to allow multiple remsh to the same node transparently
REMSH_NAME_ARG="$REMSH_TYPE remsh`date +%s`@`echo $REMSH_NAME | awk -F@ '{print $2}'`"
REMSH_REMSH_ARG="-remsh $REMSH_NAME"

# Extract the target cookie
COOKIE_ARG=`grep '^-setcookie' $VMARGS_PATH`
if [ -z "$COOKIE_ARG" ]; then
    echo "vm.args needs to have a -setcookie parameter."
    exit 1
fi

# Add ERTS bin dir to our path
ERTS_PATH=$RUNNER_BASE_DIR/erts-$ERTS_VSN/bin

# Setup command to control the node
NODETOOL="$ERTS_PATH/escript $ERTS_PATH/nodetool $NAME_ARG $COOKIE_ARG"

# Setup remote shell command to control node
REMSH="$ERTS_PATH/erl $REMSH_NAME_ARG $REMSH_REMSH_ARG $COOKIE_ARG"

case "$1" in
    dump-mnesia-data|load-mnesia-data)
        if [ "$NODE_TYPE" != leo_manager ]; then
            echo "Only leo_manager supports {dump-mnesia-data|load-mnesia-data} commands"
            exit 1
        fi
    ;;  # actual implementation follows
esac


# Check the first argument for instructions
case "$1" in
    start)
        # Make sure there is not already a node running
        RES=`$NODETOOL ping`
        if [ "$RES" = "pong" ]; then
            echo "Node is already running!"
            exit 1
        fi

        HEART_COMMAND="$RUNNER_BASE_DIR/bin/$SCRIPT start"
        export HEART_COMMAND

        # Uncomment the below two lines ONLY if you face leo_(manager|storage|gateway) restarted frequently
        # in order to dig down further for identifying the root cause.
        # ERL_CRASH_DUMP_SECONDS=-1 means setting the environment variable to a negative value
        # does not reboot the runtime system until the crash dump file is completely written.
        ## ERL_CRASH_DUMP_SECONDS=-1
        ## export ERL_CRASH_DUMP_SECONDS
        mkdir -p $PIPE_DIR
        shift # remove $1
        $ERTS_PATH/run_erl -daemon $PIPE_DIR $RUNNER_LOG_DIR "exec $RUNNER_BASE_DIR/bin/$SCRIPT console $@" 2>&1
        ;;

    stop)
        # Wait for the node to completely stop...
        case `uname -s` in
            Linux)
                # Only look for processes launched with the same NODE_EXTRA_NAME env variable
                # This will fail (won't find anything) when stopping nodes launched by older version of script
                # PID COMMAND+ENVIRONMENT
                PID=`ps axeww o pid,args|\
                    grep "NODE_EXTRA_NAME=$NODE_EXTRA_NAME " |\
                    grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $1}'`
                ;;
            Darwin|FreeBSD|DragonFly|NetBSD|OpenBSD)
                # PID COMMAND
                PID=`ps ax -o pid= -o command=|\
                    grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $1}'`
                ;;
            SunOS)
                # PID COMMAND
                PID=`ps -ef -o pid= -o args=|\
                    grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $1}'`
                ;;
            CYGWIN*)
                # UID PID PPID TTY STIME COMMAND
                PID=`ps -efW|grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $2}'`
                ;;
        esac
        $NODETOOL stop
{'fraction_non_alphanumeric': 0.11460790518382141, 'fraction_numerical': 0.004257522582129912, 'mean_word_length': 2.7477360931435966, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1417427', 'n_tokens_mistral': 6357, 'n_tokens_neox': 6052, 'n_words': 1479}