<commit_before>#!../../../../virtualenv/bin/python2.7
# -*- coding: utf-8 -*-

# NB: The shebang line above assumes you've installed a python virtual environment alongside your working copy of the
# <4most-4gp-scripts> git repository. It also only works if you invoke this python script from the directory where it
# is located. If these two assumptions are incorrect (e.g. you're using Conda), you can still use this script by typing
# <python launcher.py>, but <./launcher.py> will not work.

"""

Take a shell script which runs a load of tests through the Cannon, using <cannon_test.py>, and run all the tests in
parallel using the SLURM job management system on lunarc.

We parse the contents of the shell script, removing any commands which aren't running python, and piecing together
python commands which have been split over many lines.

Owing to aurora/lunarc being a temperamental heap of junk, it is very necessary to run this in exclusive mode, otherwise
the out of memory killer tends to kill your jobs. Well, it probably will do that anyway.

"""

import argparse
import os
import re
import logging

logging.basicConfig(level=logging.INFO, format='[%(asctime)s] %(levelname)s:%(filename)s:%(message)s',
                    datefmt='%d/%m/%Y %H:%M:%S')
logger = logging.getLogger(__name__)

# Read input parameters
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--job_list', action="append", dest='jobs',
                    help="List of shell scripts containing python commands which run tests on the Cannon.")
args = parser.parse_args()

uid = os.getpid()

slurm_script = """#!/bin/sh
# requesting the number of nodes needed
#SBATCH -N 1
#SBATCH --exclusive
#
# job time, change for what your job requires
#SBATCH -t 150:00:00
#
# job name and output file names
#SBATCH -J cannon_{destination_name}
#SBATCH -o stdout_cannon_{destination_name}_%j.out
#SBATCH -e stderr_cannon_{destination_name}_%j.out
cat $0

module add GCC/4.9.3-binutils-2.25  OpenMPI/1.8.8 CFITSIO/3.38  GCCcore/6.4.0 SQLite/3.20.1 Anaconda2

source activate myenv

cd {python_directory}
echo Starting rsync: `date`
echo Temporary directory: {temporary_directory}
mkdir {temporary_directory}
{rsync_commands}
echo Rsync done: `date`
python cannon_test.py --workspace "{temporary_directory}" {python_arguments}

"""

os.system("mkdir -p ../../output_data/cannon")

counter = 0
for job in args.jobs:
    # This is the parent directory of where the shell script is. If the shell script is in the <examples> subdirectory
    # of <test_cannon>, then we find the <cannon_test.py> python script up one level
    config_path = os.path.split(os.path.abspath(job + "/../"))[0]

    line_buffer = ""
    destination = ""
    libraries = []

    # Start piecing together the python command line to run this job
    start_string = "python2.7 cannon_test.py"

    for line in open(job):
        line = line.strip()
        # Ignore blank lines and lines which begin with a hash
        if (len(line) == 0) or (line[0] == '#'):
            continue

        # Check for output destination
        test = re.match("--output-file \"(.*)\"", line)
        if test is not None:
            destination = test.group(1)

        # Check for spectrum libraries we need
        test = re.search("(?:--train|--test) \"([^\"\\[]*)([^\"]*)?\"", line)
        if test is not None:
            libraries.append(test.group(1))

        # Lines which end in backslashes append
        if line[-1] == "\\":
            line_buffer = "{} {}".format(line_buffer, line[:-1])
            continue

        # Put current line onto the end of line buffer
        line = "{} {}".format(line_buffer, line).strip()
        line_buffer = ""

        # Ignore shell commands which don't run Cannon tests
        if not line.startswith(start_string):
            continue

        # If file product already exists, don't need to recreate it
        destination = os.path.join(config_path, destination) + ".json"
        if os.path.exists(destination):
            print "-  Product <{}> already exists".format(destination)
            continue
        else:
            print "++ Product <{}> needs making".format(destination)

        # Create rsync command to copy spectrum libraries to node local storage
        run_id = "{}_{}".format(uid, counter)
        tmp_dir = "${{TMPDIR}}/cannon_{}".format(run_id)<commit_msg>Bugfixes to lunarc cannon launcher<commit_after>#!../../../../virtualenv/bin/python2.7
# -*- coding: utf-8 -*-

# NB: The shebang line above assumes you've installed a python virtual environment alongside your working copy of the
# <4most-4gp-scripts> git repository. It also only works if you invoke this python script from the directory where it
# is located. If these two assumptions are incorrect (e.g. you're using Conda), you can still use this script by typing
# <python launcher.py>, but <./launcher.py> will not work.

"""

Take a shell script which runs a load of tests through the Cannon, using <cannon_test.py>, and run all the tests in
parallel using the SLURM job management system on lunarc.

We parse the contents of the shell script, removing any commands which aren't running python, and piecing together
python commands which have been split over many lines.

Owing to aurora/lunarc being a temperamental heap of junk, it is very necessary to run this in exclusive mode, otherwise
the out of memory killer tends to kill your jobs. Well, it probably will do that anyway.

"""

import argparse
import os
import re
import logging

logging.basicConfig(level=logging.INFO, format='[%(asctime)s] %(levelname)s:%(filename)s:%(message)s',
                    datefmt='%d/%m/%Y %H:%M:%S')
logger = logging.getLogger(__name__)

# Read input parameters
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--job_list', action="append", dest='jobs',
                    help="List of shell scripts containing python commands which run tests on the Cannon.")
args = parser.parse_args()

uid = os.getpid()

slurm_script = """#!/bin/sh
# requesting the number of nodes needed
#SBATCH -N 1
#SBATCH --exclusive
#
# job time, change for what your job requires
#SBATCH -t 150:00:00
#
# job name and output file names
#SBATCH -J cannon_{destination_name}
#SBATCH -o stdout_cannon_{destination_name}_%j.out
#SBATCH -e stderr_cannon_{destination_name}_%j.out
cat $0

module add GCC/4.9.3-binutils-2.25  OpenMPI/1.8.8 CFITSIO/3.38  GCCcore/6.4.0 SQLite/3.20.1 Anaconda2

source activate myenv

cd {python_directory}
echo Starting rsync: `date`
echo Temporary directory: {temporary_directory}
mkdir {temporary_directory}
{rsync_commands}
echo Rsync done: `date`
python cannon_test.py --workspace "{temporary_directory}" {python_arguments}

"""

os.system("mkdir -p ../../../output_data/cannon")

counter = 0
for job in args.jobs:
    # This is the parent directory of where the shell script is. If the shell script is in the <examples> subdirectory
    # of <test_cannon>, then we find the <cannon_test.py> python script up one level
    config_path = os.path.split(os.path.abspath(job + "/../"))[0]

    line_buffer = ""
    destination = ""
    libraries = []

    # Start piecing together the python command line to run this job
    start_string = "python2.7 cannon_test.py"

    for line in open(job):
        line = line.strip()
        # Ignore blank lines and lines which begin with a hash
        if (len(line) == 0) or (line[0] == '#'):
            continue

        # Check for output destination
        test = re.match("--output-file \"(.*)\"", line)
        if test is not None:
            destination = test.group(1)

        # Check for spectrum libraries we need
        test = re.search("(?:--train|--test) \"([^\"\\[]*)([^\"]*)?\"", line)
        if test is not None:
            libraries.append(test.group(1))

        # Lines which end in backslashes append
        if line[-1] == "\\":
            line_buffer = "{} {}".format(line_buffer, line[:-1])
            continue

        # Put current line onto the end of line buffer
        line = "{} {}".format(line_buffer, line).strip()
        line_buffer = ""

        # Ignore shell commands which don't run Cannon tests
        if not line.startswith(start_string):
            continue

        # If file product already exists, don't need to recreate it
        destination = os.path.join(config_path, destination) + ".full.json.gz"
        if os.path.exists(destination):
            print "-  Product <{}> already exists".format(destination)
            continue
        else:
            print "++ Product <{}> needs making".format(destination)

        # Create rsync command to copy spectrum libraries to node local storage
        run_id = "{}_{}".format(uid, counter)
        tmp_dir = "${{TMPDIR}}/cannon_{}".format(run_id)
{'fraction_non_alphanumeric': 0.1166931276933545, 'fraction_numerical': 0.00997958720798367, 'mean_word_length': 3.094243268337976, 'pattern_counts': {'":': 2, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '494452', 'n_tokens_mistral': 2652, 'n_tokens_neox': 2552, 'n_words': 933}