<commit_before>from abc import ABCMeta, abstractmethod

from libsubmit.utils import RepresentationMixin


class Launcher(RepresentationMixin, metaclass=ABCMeta):
    """ Launcher base class to enforce launcher interface
    """
    @abstractmethod
    def __call__(self, command, tasks_per_node, nodes_per_block, walltime=None):
        """ Wraps the command with the Launcher calls.
        *MUST* be implemented by the concrete child classes
        """
        pass


class SimpleLauncher(Launcher):
    """ Does no wrapping. Just returns the command as-is
    """

    def __call__(self, command, tasks_per_node, nodes_per_block, walltime=None):
        """
        Args:
        - command (string): The command string to be launched
        - task_block (string) : bash evaluated string.

        KWargs:
        - walltime (int) : This is not used by this launcher.
        """
        return command


class SingleNodeLauncher(Launcher):
    """ Worker launcher that wraps the user's command with the framework to
    launch multiple command invocations in parallel. This wrapper sets the
    bash env variable CORES to the number of cores on the machine. By setting
    task_blocks to an integer or to a bash expression the number of invocations
    of the command to be launched can be controlled.
    """
    def __call__(self, command, tasks_per_node, nodes_per_block, walltime=None):
        """
        Args:
        - command (string): The command string to be launched
        - task_block (string) : bash evaluated string.

        KWargs:
        - walltime (int) : This is not used by this launcher.
        """
        task_blocks = tasks_per_node * nodes_per_block

        x = '''export CORES=$(getconf _NPROCESSORS_ONLN)
echo "Found cores : $CORES"
WORKERCOUNT={1}

CMD ( ) {{
{0}
}}
for COUNT in $(seq 1 1 $WORKERCOUNT)
do
    echo "Launching worker: $COUNT"
    CMD &
done
wait
echo "All workers done"
'''.format(command, task_blocks)
        return x


class GnuParallelLauncher(Launcher):
    """ Worker launcher that wraps the user's command with the framework to
    launch multiple command invocations via GNU parallel sshlogin.

    This wrapper sets the bash env variable CORES to the number of cores on the
    machine.

    This launcher makes the following assumptions:
    - GNU parallel is installed and can be located in $PATH
    - Paswordless SSH login is configured between the controller node and the
      target nodes.
    - The provider makes available the $PBS_NODEFILE environment variable
    """
    def __call__(self, command, tasks_per_node, nodes_per_block, walltime=None):
        """
        Args:
        - command (string): The command string to be launched
        - task_block (string) : bash evaluated string.

        KWargs:
        - walltime (int) : This is not used by this launcher.
        """
        task_blocks = tasks_per_node * nodes_per_block

        x = '''export CORES=$(getconf _NPROCESSORS_ONLN)
echo "Found cores : $CORES"
WORKERCOUNT={3}

# Deduplicate the nodefile
SSHLOGINFILE="$JOBNAME.nodes"
if [ -z "$PBS_NODEFILE" ]; then
    echo "localhost" > $SSHLOGINFILE
else
    sort -u $PBS_NODEFILE > $SSHLOGINFILE
fi

cat << PARALLEL_CMD_EOF > cmd_$JOBNAME.sh
{0}
PARALLEL_CMD_EOF
chmod u+x cmd_$JOBNAME.sh

#file to contain the commands to parallel
PFILE=cmd_${{JOBNAME}}.sh.parallel

# Truncate the file
cp /dev/null $PFILE

for COUNT in $(seq 1 1 $WORKERCOUNT)
do
    echo "sh cmd_$JOBNAME.sh" >> $PFILE
done

parallel --env _ --joblog "$JOBNAME.sh.parallel.log" \
    --sshloginfile $SSHLOGINFILE --jobs {1} < $PFILE

echo "All workers done"
'''.format(command, tasks_per_node, nodes_per_block, task_blocks)
        return x


class MpiExecLauncher(Launcher):
    """ Worker launcher that wraps the user's command with the framework to
    launch multiple command invocations via mpiexec.

    This wrapper sets the bash env variable CORES to the number of cores on the
    machine.

    This launcher makes the following assumptions:
    - mpiexec is installed and can be located in $PATH
    - The provider makes available the $PBS_NODEFILE environment variable
    """
    def __call__(self, command, tasks_per_node, nodes_per_block, walltime=None):
        """
        Args:
        - command (string): The command string to be launched
        - task_block (string) : bash evaluated string.

        KWargs:
        - walltime (int) : This is not used by this launcher.
        """
        task_blocks = tasks_per_node * nodes_per_block

        x = '''export CORES=$(getconf _NPROCESSORS_ONLN)
echo "Found cores : $CORES"
WORKERCOUNT={3}

# Deduplicate the nodefile
HOSTFILE="$JOBNAME.nodes"
if [ -z "$HOSTFILE" ]; then
    echo "localhost" > $HOSTFILE
else
    sort -u $PBS_NODEFILE > $HOSTFILE
fi

cat << MPIEXEC_EOF > cmd_$JOBNAME.sh
{0}
MPIEXEC_EOF
chmod u+x cmd_$JOBNAME.sh

mpiexec --bind-to none -n $WORKERCOUNT --hostfile $HOSTFILE /usr/bin/sh cmd_$JOBNAME.sh

echo "All workers done"<commit_msg>Revert "Fix variable name."<commit_after>from abc import ABCMeta, abstractmethod

from libsubmit.utils import RepresentationMixin


class Launcher(RepresentationMixin, metaclass=ABCMeta):
    """ Launcher base class to enforce launcher interface
    """
    @abstractmethod
    def __call__(self, command, tasks_per_node, nodes_per_block, walltime=None):
        """ Wraps the command with the Launcher calls.
        *MUST* be implemented by the concrete child classes
        """
        pass


class SimpleLauncher(Launcher):
    """ Does no wrapping. Just returns the command as-is
    """

    def __call__(self, command, tasks_per_node, nodes_per_block, walltime=None):
        """
        Args:
        - command (string): The command string to be launched
        - task_block (string) : bash evaluated string.

        KWargs:
        - walltime (int) : This is not used by this launcher.
        """
        return command


class SingleNodeLauncher(Launcher):
    """ Worker launcher that wraps the user's command with the framework to
    launch multiple command invocations in parallel. This wrapper sets the
    bash env variable CORES to the number of cores on the machine. By setting
    task_blocks to an integer or to a bash expression the number of invocations
    of the command to be launched can be controlled.
    """
    def __call__(self, command, tasks_per_node, nodes_per_block, walltime=None):
        """
        Args:
        - command (string): The command string to be launched
        - task_block (string) : bash evaluated string.

        KWargs:
        - walltime (int) : This is not used by this launcher.
        """
        task_blocks = tasks_per_node * nodes_per_block

        x = '''export CORES=$(getconf _NPROCESSORS_ONLN)
echo "Found cores : $CORES"
WORKERCOUNT={1}

CMD ( ) {{
{0}
}}
for COUNT in $(seq 1 1 $WORKERCOUNT)
do
    echo "Launching worker: $COUNT"
    CMD &
done
wait
echo "All workers done"
'''.format(command, task_blocks)
        return x


class GnuParallelLauncher(Launcher):
    """ Worker launcher that wraps the user's command with the framework to
    launch multiple command invocations via GNU parallel sshlogin.

    This wrapper sets the bash env variable CORES to the number of cores on the
    machine.

    This launcher makes the following assumptions:
    - GNU parallel is installed and can be located in $PATH
    - Paswordless SSH login is configured between the controller node and the
      target nodes.
    - The provider makes available the $PBS_NODEFILE environment variable
    """
    def __call__(self, command, tasks_per_node, nodes_per_block, walltime=None):
        """
        Args:
        - command (string): The command string to be launched
        - task_block (string) : bash evaluated string.

        KWargs:
        - walltime (int) : This is not used by this launcher.
        """
        task_blocks = tasks_per_node * nodes_per_block

        x = '''export CORES=$(getconf _NPROCESSORS_ONLN)
echo "Found cores : $CORES"
WORKERCOUNT={3}

# Deduplicate the nodefile
SSHLOGINFILE="$JOBNAME.nodes"
if [ -z "$PBS_NODEFILE" ]; then
    echo "localhost" > $SSHLOGINFILE
else
    sort -u $PBS_NODEFILE > $SSHLOGINFILE
fi

cat << PARALLEL_CMD_EOF > cmd_$JOBNAME.sh
{0}
PARALLEL_CMD_EOF
chmod u+x cmd_$JOBNAME.sh

#file to contain the commands to parallel
PFILE=cmd_${{JOBNAME}}.sh.parallel

# Truncate the file
cp /dev/null $PFILE

for COUNT in $(seq 1 1 $WORKERCOUNT)
do
    echo "sh cmd_$JOBNAME.sh" >> $PFILE
done

parallel --env _ --joblog "$JOBNAME.sh.parallel.log" \
    --sshloginfile $SSHLOGINFILE --jobs {1} < $PFILE

echo "All workers done"
'''.format(command, tasks_per_node, nodes_per_block, task_blocks)
        return x


class MpiExecLauncher(Launcher):
    """ Worker launcher that wraps the user's command with the framework to
    launch multiple command invocations via mpiexec.

    This wrapper sets the bash env variable CORES to the number of cores on the
    machine.

    This launcher makes the following assumptions:
    - mpiexec is installed and can be located in $PATH
    - The provider makes available the $PBS_NODEFILE environment variable
    """
    def __call__(self, command, tasks_per_node, nodes_per_block, walltime=None):
        """
        Args:
        - command (string): The command string to be launched
        - task_block (string) : bash evaluated string.

        KWargs:
        - walltime (int) : This is not used by this launcher.
        """
        task_blocks = tasks_per_node * nodes_per_block

        x = '''export CORES=$(getconf _NPROCESSORS_ONLN)
echo "Found cores : $CORES"
WORKERCOUNT={3}

# Deduplicate the nodefile
HOSTFILE="$JOBNAME.nodes"
if [ -z "$PBS_NODEFILE" ]; then
    echo "localhost" > $HOSTFILE
else
    sort -u $PBS_NODEFILE > $HOSTFILE
fi

cat << MPIEXEC_EOF > cmd_$JOBNAME.sh
{0}
MPIEXEC_EOF
chmod u+x cmd_$JOBNAME.sh

mpiexec --bind-to none -n $WORKERCOUNT --hostfile $HOSTFILE /usr/bin/sh cmd_$JOBNAME.sh

echo "All workers done"
{'fraction_non_alphanumeric': 0.08491698339667933, 'fraction_numerical': 0.0022004400880176033, 'mean_word_length': 3.0762331838565022, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3200607', 'n_tokens_mistral': 3111, 'n_tokens_neox': 2987, 'n_words': 1072}