<commit_before>#!/usr/bin/python

import sys
import os
import shutil
import signal
import time
import random
import getpass
import re
import subprocess
import ConfigParser


def is_python_test_file(file_name):
    """
    Return True if file_name matches a regexp for a python test.  False otherwise.
    """

    if (file_name == "test_config.py"):
        return False

    if re.match("^pyunit.*\.py$", file_name):
        return True

    if (re.match("^test.*\.py$", file_name)):
        return True

    return False


def is_python_file(file_name):
    """
    Return True if file_name matches a regexp for a python program in general.  False otherwise.

    This is a separate function because it's useful to have the scan-for-test operation in
    build_test_list() be separated from running the test.

    That allows us to run things explictly named using the --test option.  Such as:
        run.py --wipeall --numclouds 1 --test generate_rest_api_docs.py
    """

    if (file_name == "test_config.py"):
        return False

    if (re.match("^.*\.py$", file_name)):
        return True

    return False


def is_javascript_test_file(file_name):
    """
    Return True if file_name matches a regexp for a javascript test.  False otherwise.
    """

    if (re.match("^.*test.*\.js$", file_name)):
        return True

    return False


def is_runit_test_file(file_name):
    """
    Return True if file_name matches a regexp for a R test.  False otherwise.
    """

    if (file_name == "h2o-runit.R"):
        return False

    if (re.match("^runit.*\.[rR]$", file_name)):
        return True

    return False


class H2OUseCloudNode:
    """
    A class representing one node in an H2O cloud which was specified by the user.
    Don't try to build or tear down this kind of node.

    use_ip: The given ip of the cloud.
    use_port: The given port of the cloud.
    """

    def __init__(self, use_ip, use_port):
        self.use_ip = use_ip
        self.use_port = use_port

    def start(self):
        pass

    def stop(self):
        pass

    def terminate(self):
        pass

    def get_ip(self):
        return self.use_ip

    def get_port(self):
        return self.use_port


class H2OUseCloud:
    """
    A class representing an H2O clouds which was specified by the user.
    Don't try to build or tear down this kind of cloud.
    """

    def __init__(self, cloud_num, use_ip, use_port):
        self.cloud_num = cloud_num
        self.use_ip = use_ip
        self.use_port = use_port

        self.nodes = []
        node = H2OUseCloudNode(self.use_ip, self.use_port)
        self.nodes.append(node)

    def start(self):
        pass

    def wait_for_cloud_to_be_up(self):
        pass

    def stop(self):
        pass

    def terminate(self):
        pass

    def get_ip(self):
        node = self.nodes[0]
        return node.get_ip()

    def get_port(self):
        node = self.nodes[0]
        return node.get_port()


class H2OCloudNode:
    """
    A class representing one node in an H2O cloud.
    Note that the base_port is only a request for H2O.
    H2O may choose to ignore our request and pick any port it likes.
    So we have to scrape the real port number from stdout as part of cloud startup.

    port: The actual port chosen at run time.
    pid: The process id of the node.
    output_file_name: Where stdout and stderr go.  They are merged.
    child: subprocess.Popen object.
    terminated: Only from a signal.  Not normal shutdown.
    """

    def __init__(self, is_client, cloud_num, nodes_per_cloud, node_num, cloud_name, h2o_jar, ip, base_port,
                 xmx, output_dir):
        """
        Create a node in a cloud.

        @param is_client: Whether this node is an H2O client node (vs a worker node) or not.
        @param cloud_num: Dense 0-based cloud index number.
        @param nodes_per_cloud: How many H2O java instances are in a cloud.  Clouds are symmetric.
        @param node_num: This node's dense 0-based node index number.
        @param cloud_name: The H2O -name command-line argument.
        @param h2o_jar: Path to H2O jar file.
        @param base_port: The starting port number we are trying to get our nodes to listen on.
        @param xmx: Java memory parameter.
        @param output_dir: The directory where we can create an output file for this process.
        @return: The node object.
        """
        self.is_client = is_client
        self.cloud_num = cloud_num
        self.nodes_per_cloud = nodes_per_cloud
        self.node_num = node_num
        self.cloud_name = cloud_name
        self.h2o_jar = h2o_jar
        self.ip = ip
        self.base_port = base_port
        self.xmx = xmx
        self.output_dir = output_dir

        self.port = -1
        self.pid = -1
        self.output_file_name = ""
        self.child = None
        self.terminated = False

        # Choose my base port number here.  All math is done here.  Every node has the same
        # base_port and calculates it's own my_base_port.
        ports_per_node = 2
        self.my_base_port = \
            self.base_port + \
            (self.cloud_num * self.nodes_per_cloud * ports_per_node) + \
            (self.node_num * ports_per_node)

    def start(self):
        """
        Start one node of H2O.
        (Stash away the self.child and self.pid internally here.)

        @return: none
        """

        # there is no hdfs currently in ec2, except s3n/hdfs
        # the core-site.xml provides s3n info
        # it's possible that we can just always hardware the hdfs version
        # to match the cdh3 cluster we're hardwiring tests to
        # i.e. it won't make s3n/s3 break on ec2

        if (self.is_client):
            main_class = "water.H2OClientApp"
        else:
            main_class = "water.H2OApp"
        if "JAVA_HOME" in os.environ:
            java = os.environ["JAVA_HOME"] + "/bin/java"
        else:
            java = "java"
        cmd = [java,
               # "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005",
               "-Xmx" + self.xmx,
               "-ea",
               "-cp", self.h2o_jar,
               main_class,
               "-name", self.cloud_name,
               "-baseport", str(self.my_base_port)]

        # Add S3N credentials to cmd if they exist.
        # ec2_hdfs_config_file_name = os.path.expanduser("~/.ec2/core-site.xml")
        # if (os.path.exists(ec2_hdfs_config_file_name)):
        #     cmd.append("-hdfs_config")
        #     cmd.append(ec2_hdfs_config_file_name)

        self.output_file_name = \
            os.path.join(self.output_dir, "java_" + str(self.cloud_num) + "_" + str(self.node_num) + ".out.txt")
        f = open(self.output_file_name, "w")

        if g_convenient:
            cwd = os.getcwd()
            here = os.path.abspath(os.path.dirname(__file__))
            there = os.path.abspath(os.path.join(here, ".."))
            os.chdir(there)
            self.child = subprocess.Popen(args=cmd,
                                          stdout=f,
                                          stderr=subprocess.STDOUT,
                                          cwd=there)
            os.chdir(cwd)
        else:
            self.child = subprocess.Popen(args=cmd,
                                          stdout=f,
                                          stderr=subprocess.STDOUT,
                                          cwd=self.output_dir)

        self.pid = self.child.pid
        print("+ CMD: " + ' '.join(cmd))

    def scrape_port_from_stdout(self):
        """
        Look at the stdout log and figure out which port the JVM chose.
        Write this to self.port.
        This call is blocking.
        Exit if this fails.

        @return: none
        """
        retries = 30
        while (retries > 0):
            if (self.terminated):
                return
            f = open(self.output_file_name, "r")
            s = f.readline()
            while (len(s) > 0):
                if (self.terminated):
                    return
                match_groups = re.search(r"Listening for HTTP and REST traffic on  http://(\S+):(\d+)", s)
                if (match_groups is not None):
                    port = match_groups.group(2)
                    if (port is not None):
                        self.port = port
                        f.close()
                        print("H2O Cloud {} Node {} started with output file {}".format(self.cloud_num,
                                                                                        self.node_num,
                                                                                        self.output_file_name))
                        return

                s = f.readline()

            f.close()
            retries -= 1
            if (self.terminated):
                return
            time.sleep(1)

        print("")
        print("ERROR: Too many retries starting cloud.")
        print("")
        sys.exit(1)

    def scrape_cloudsize_from_stdout(self, nodes_per_cloud):
        """
        Look at the stdout log and wait until the cloud of proper size is formed.
        This call is blocking.
        Exit if this fails.

        @return: none
        """
        retries = 60
        while (retries > 0):
            if (self.terminated):
                return
            f = open(self.output_file_name, "r")
            s = f.readline()
            while (len(s) > 0):
                if (self.terminated):
                    return
                match_groups = re.search(r"Cloud of size (\d+) formed", s)
                if (match_groups is not None):
                    size = match_groups.group(1)
                    if (size is not None):
                        size = int(size)
                        if (size == nodes_per_cloud):
                            f.close()
                            return

                s = f.readline()

            f.close()
            retries -= 1
            if (self.terminated):
                return
            time.sleep(1)

        print("")
        print("ERROR: Too many retries starting cloud.")
        print("")
        sys.exit(1)

    def stop(self):
        """
        Normal node shutdown.
        Ignore failures for now.

        @return: none
        """
        if (self.pid > 0):
            print("Killing JVM with PID {}".format(self.pid))
            try:
                self.child.terminate()
                self.child.wait()
            except OSError:
                pass
            self.pid = -1

    def terminate(self):
        """
        Terminate a running node.  (Due to a signal.)

        @return: none
        """
        self.terminated = True
        self.stop()

    def get_ip(self):
        """ Return the ip address this node is really listening on. """
        return self.ip

    def get_port(self):
        """ Return the port this node is really listening on. """
        return self.port

    def __str__(self):
        s = ""
        s += "    node {}\n".format(self.node_num)
        s += "        xmx:          {}\n".format(self.xmx)
        s += "        my_base_port: {}\n".format(self.my_base_port)
        s += "        port:         {}\n".format(self.port)
        s += "        pid:          {}\n".format(self.pid)
        return s


class H2OCloud:
    """
    A class representing one of the H2O clouds.
    """

    def __init__(self, cloud_num, use_client, nodes_per_cloud, h2o_jar, base_port, xmx, output_dir):
        """
        Create a cloud.
        See node definition above for argument descriptions.

        @return: The cloud object.
        """
        self.use_client = use_client
        self.cloud_num = cloud_num
        self.nodes_per_cloud = nodes_per_cloud
        self.h2o_jar = h2o_jar
        self.base_port = base_port
        self.xmx = xmx
        self.output_dir = output_dir

        # Randomly choose a seven digit cloud number.
        n = random.randint(1000000, 9999999)
        user = getpass.getuser()
        user = ''.join(user.split())

        self.cloud_name = "H2O_runit_{}_{}".format(user, n)
        self.nodes = []
        self.client_nodes = []
        self.jobs_run = 0

        if (use_client):
            actual_nodes_per_cloud = self.nodes_per_cloud + 1
        else:
            actual_nodes_per_cloud = self.nodes_per_cloud

        for node_num in range(actual_nodes_per_cloud):
            is_client = False
            if (use_client):
                if (node_num == (actual_nodes_per_cloud - 1)):
                    is_client = True
            node = H2OCloudNode(is_client,
                                self.cloud_num, actual_nodes_per_cloud, node_num,
                                self.cloud_name,
                                self.h2o_jar,
                                "127.0.0.1", self.base_port,
                                self.xmx, self.output_dir)
            if (is_client):
                self.client_nodes.append(node)
            else:
                self.nodes.append(node)

    def start(self):
        """
        Start H2O cloud.
        The cloud is not up until wait_for_cloud_to_be_up() is called and returns.

        @return: none
        """
        for node in self.nodes:
            node.start()

        for node in self.client_nodes:
            node.start()

    def wait_for_cloud_to_be_up(self):
        """
        Blocking call ensuring the cloud is available.

        @return: none
        """
        self._scrape_port_from_stdout()
        self._scrape_cloudsize_from_stdout()

    def stop(self):
        """
        Normal cloud shutdown.

        @return: none
        """
        for node in self.nodes:
            node.stop()

        for node in self.client_nodes:
            node.stop()

    def terminate(self):
        """
        Terminate a running cloud.  (Due to a signal.)

        @return: none
        """
        for node in self.client_nodes:
            node.terminate()

        for node in self.nodes:
            node.terminate()

    def get_ip(self):
        """ Return an ip to use to talk to this cloud. """
        if (len(self.client_nodes) > 0):
            node = self.client_nodes[0]
        else:
            node = self.nodes[0]
        return node.get_ip()

    def get_port(self):
        """ Return a port to use to talk to this cloud. """
        if (len(self.client_nodes) > 0):
            node = self.client_nodes[0]
        else:
            node = self.nodes[0]
        return node.get_port()

    def _scrape_port_from_stdout(self):
        for node in self.nodes:
            node.scrape_port_from_stdout()
        for node in self.client_nodes:
            node.scrape_port_from_stdout()

    def _scrape_cloudsize_from_stdout(self):
        for node in self.nodes:
            node.scrape_cloudsize_from_stdout(self.nodes_per_cloud)
        for node in self.client_nodes:
            node.scrape_cloudsize_from_stdout(self.nodes_per_cloud)

    def __str__(self):
        s = ""
        s += "cloud {}\n".format(self.cloud_num)
        s += "    name:     {}\n".format(self.cloud_name)
        s += "    jobs_run: {}\n".format(self.jobs_run)
        for node in self.nodes:
            s += str(node)
        for node in self.client_nodes:
            s += str(node)
        return s


class Test:
    """
    A class representing one Test.

    cancelled: Don't start this test.
    terminated: Test killed due to signal.
    returncode: Exit code of child.
    pid: Process id of the test.
    ip: IP of cloud to run test.
    port: Port of cloud to run test.
    child: subprocess.Popen object.
    """

    @staticmethod
    def test_did_not_complete():
        """
        returncode marker to know if the test ran or not.
        """
        return -9999999

    def __init__(self, test_dir, test_short_dir, test_name, output_dir):
        """
        Create a Test.

        @param test_dir: Full absolute path to the test directory.
        @param test_short_dir: Path from h2o/R/tests to the test directory.
        @param test_name: Test filename with the directory removed.
        @param output_dir: The directory where we can create an output file for this process.
        @return: The test object.
        """
        self.test_dir = test_dir
        self.test_short_dir = test_short_dir
        self.test_name = test_name
        self.output_dir = output_dir
        self.output_file_name = ""

        self.cancelled = False
        self.terminated = False
        self.returncode = Test.test_did_not_complete()
        self.start_seconds = -1
        self.pid = -1
        self.ip = None
        self.port = -1
        self.child = None

    def start(self, ip, port):
        """
        Start the test in a non-blocking fashion.

        @param ip: IP address of cloud to run on.
        @param port: Port of cloud to run on.
        @return: none
        """
        if (self.cancelled or self.terminated):
            return

        self.start_seconds = time.time()
        self.ip = ip
        self.port = port

        if (is_python_test_file(self.test_name)):
            cmd = ["python",
                   self.test_name,
                   "--usecloud",
                   self.ip + ":" + str(self.port)]
        elif (is_python_file(self.test_name)):
            cmd = ["python",
                   self.test_name,
                   "--usecloud",
                   self.ip + ":" + str(self.port)]
        elif (is_runit_test_file(self.test_name)):
            cmd = ["R",
                   "-f",
                   self.test_name,
                   "--args",
                   self.ip + ":" + str(self.port)]
        elif (is_javascript_test_file(self.test_name)):
            cmd = ["phantomjs",
                   self.test_name,
                   self.ip + ":" + str(self.port)]
        else:
            print("")
            print("ERROR: Test runner failure with test: " + self.test_name)
            print("")
            sys.exit(1)

        test_short_dir_with_no_slashes = re.sub(r'[\\/]', "_", self.test_short_dir)
        if (len(test_short_dir_with_no_slashes) > 0):
            test_short_dir_with_no_slashes += "_"
        self.output_file_name = \
            os.path.join(self.output_dir, test_short_dir_with_no_slashes + self.test_name + ".out.txt")
        f = open(self.output_file_name, "w")
        self.child = subprocess.Popen(args=cmd,
                                      stdout=f,
                                      stderr=subprocess.STDOUT,
                                      cwd=self.test_dir)
        self.pid = self.child.pid
        # print("+ CMD: " + ' '.join(cmd))

    def is_completed(self):
        """
        Check if test has completed.

        This has side effects and MUST be called for the normal test queueing to work.
        Specifically, child.poll().

        @return: True if the test completed, False otherwise.
        """
        child = self.child
        if (child is None):
            return False
        child.poll()
        if (child.returncode is None):
            return False
        self.pid = -1
        self.returncode = child.returncode
        return True

    def cancel(self):
        """
        Mark this test as cancelled so it never tries to start.

        @return: none
        """
        if (self.pid <= 0):
            self.cancelled = True

    def terminate_if_started(self):
        """
        Terminate a running test.  (Due to a signal.)

        @return: none
        """
        if (self.pid > 0):
            self.terminate()

    def terminate(self):
        """
        Terminate a running test.  (Due to a signal.)

        @return: none
        """
        self.terminated = True
        if (self.pid > 0):
            print("Killing Test {} with PID {}".format(os.path.join(self.test_short_dir, self.test_name), self.pid))
            try:
                self.child.terminate()
            except OSError:
                pass
        self.pid = -1

    def get_test_dir_file_name(self):
        """
        @return: The full absolute path of this test.
        """
        return os.path.join(self.test_dir, self.test_name)

    def get_test_name(self):
        """
        @return: The file name (no directory) of this test.
        """
        return self.test_name

    def get_seed_used(self):
        """
        @return: The seed used by this test.
        """
        return self._scrape_output_for_seed()

    def get_ip(self):
        """
        @return: IP of the cloud where this test ran.
        """
        return self.ip

    def get_port(self):
        """
        @return: Integer port number of the cloud where this test ran.
        """
        return int(self.port)

    def get_passed(self):
        """
        @return: True if the test passed, False otherwise.
        """
        return (self.returncode == 0)

    def get_nopass(self, nopass):
        """
        Some tests are known not to fail and even if they don't pass we don't want
        to fail the overall regression PASS/FAIL status.

        @return: True if the test has been marked as NOPASS, False otherwise.
        """
        a = re.compile("NOPASS")
        return a.search(self.test_name) and not nopass

    def get_nofeature(self, nopass):
        """
        Some tests are known not to fail and even if they don't pass we don't want
        to fail the overall regression PASS/FAIL status.

        @return: True if the test has been marked as NOFEATURE, False otherwise.
        """
        a = re.compile("NOFEATURE")
        return a.search(self.test_name) and not nopass

    def get_completed(self):
        """
        @return: True if the test completed (pass or fail), False otherwise.
        """
        return (self.returncode > Test.test_did_not_complete())

    def get_terminated(self):
        """
        For a test to be terminated it must have started and had a PID.

        @return: True if the test was terminated, False otherwise.
        """
        return self.terminated

    def get_output_dir_file_name(self):
        """
        @return: Full path to the output file which you can paste to a terminal window.
        """
        return (os.path.join(self.output_dir, self.output_file_name))

    def _scrape_output_for_seed(self):
        """
        @return: The seed scraped from the outpul file.
        """
        res = ""
        with open(self.get_output_dir_file_name(), "r") as f:
            for line in f:
                if "SEED used" in line:
                    line = line.strip().split(' ')
                    res = line[-1]
                    break
        return res

    def __str__(self):
        s = ""
        s += "Test: {}/{}\n".format(self.test_dir, self.test_name)
        return s


class TestRunner:
    """
    A class for running tests.

    The tests list contains an object for every test.
    The tests_not_started list acts as a job queue.
    The tests_running list is polled for jobs that have finished.
    """

    def __init__(self,
                 test_root_dir,
                 use_cloud, use_cloud2, use_client, cloud_config, use_ip, use_port,
                 num_clouds, nodes_per_cloud, h2o_jar, base_port, xmx, output_dir,
                 failed_output_dir, path_to_tar, path_to_whl, produce_unit_reports, testreport_dir):
        """
        Create a runner.

        @param test_root_dir: h2o/R/tests directory.
        @param use_cloud: Use this one user-specified cloud.  Overrides num_clouds.
        @param use_cloud2: Use the cloud_config to define the list of H2O clouds.
        @param cloud_config: (if use_cloud2) the config file listing the H2O clouds.
        @param use_ip: (if use_cloud) IP of one cloud to use.
        @param use_port: (if use_cloud) Port of one cloud to use.
        @param num_clouds: Number of H2O clouds to start.
        @param nodes_per_cloud: Number of H2O nodes to start per cloud.
        @param h2o_jar: Path to H2O jar file to run.
        @param base_port: Base H2O port (e.g. 54321) to start choosing from.
        @param xmx: Java -Xmx parameter.
        @param output_dir: Directory for output files.
        @param failed_output_dir: Directory to copy failed test output.
        @param path_to_tar: NA
        @param path_to_whl: NA
        @param produce_unit_reports: if true then runner produce xUnit test reports for Jenkins
        @param testreport_dir: directory to put xUnit test reports for Jenkins (should follow build system conventions)
        @return: The runner object.
        """
        self.test_root_dir = test_root_dir

        self.use_cloud = use_cloud
        self.use_cloud2 = use_cloud2
        self.use_client = use_client

        # Valid if use_cloud is True
        self.use_ip = use_ip
        self.use_port = use_port

        # Valid if use_cloud is False
        self.num_clouds = num_clouds
        self.nodes_per_cloud = nodes_per_cloud
        self.h2o_jar = h2o_jar
        self.base_port = base_port
        self.output_dir = output_dir
        self.failed_output_dir = failed_output_dir
        self.produce_unit_reports = produce_unit_reports
        self.testreport_dir = testreport_dir

        self.start_seconds = time.time()
        self.terminated = False
        self.clouds = []
        self.tests = []
        self.tests_not_started = []
        self.tests_running = []
        self.regression_passed = False
        self._create_output_dir()
        self._create_failed_output_dir()
        if produce_unit_reports:
            self._create_testreport_dir()
        self.nopass_counter = 0
        self.nofeature_counter = 0
        self.path_to_tar = path_to_tar
        self.path_to_whl = path_to_whl

        if (use_cloud):
            node_num = 0
            cloud = H2OUseCloud(node_num, use_ip, use_port)
            self.clouds.append(cloud)
        elif (use_cloud2):
            clouds = TestRunner.read_config(cloud_config)
            node_num = 0
            for c in clouds:
                cloud = H2OUseCloud(node_num, c[0], c[1])
                self.clouds.append(cloud)
                node_num += 1
        else:
            for i in range(self.num_clouds):
                cloud = H2OCloud(i, self.use_client, self.nodes_per_cloud, h2o_jar, self.base_port, xmx,
                                 self.output_dir)
                self.clouds.append(cloud)

    @staticmethod
    def find_test(test_to_run):
        """
        Be nice and try to help find the test if possible.
        If the test is actually found without looking, then just use it.
        Otherwise, search from the script's down directory down.
        """
        if (os.path.exists(test_to_run)):
            abspath_test = os.path.abspath(test_to_run)
            return abspath_test

        for d, subdirs, files in os.walk(os.getcwd()):
            for f in files:
                if (f == test_to_run):
                    return os.path.join(d, f)

        # Not found, return the file, which will result in an error downstream when it can't be found.
        print("")
        print("ERROR: Test does not exist: " + test_to_run)
        print("")
        sys.exit(1)

    @staticmethod
    def read_config(config_file):
        clouds = []  # a list of lists. Inner lists have [node_num, ip, port]
        cfg = ConfigParser.RawConfigParser()
        cfg.read(config_file)
        for s in cfg.sections():
            items = cfg.items(s)
            cloud = [items[0][1], int(items[1][1])]
            clouds.append(cloud)
        return clouds

    def read_test_list_file(self, test_list_file):
        """
        Read in a test list file line by line.  Each line in the file is a test
        to add to the test run.

        @param test_list_file: Filesystem path to a file with a list of tests to run.
        @return: none
        """
        try:
            f = open(test_list_file, "r")
            s = f.readline()
            while (len(s) != 0):
                stripped = s.strip()
                if (len(stripped) == 0):
                    s = f.readline()
                    continue
                if (stripped.startswith("#")):
                    s = f.readline()
                    continue
                found_stripped = TestRunner.find_test(stripped)
                self.add_test(found_stripped)
                s = f.readline()
            f.close()
        except IOError as e:
            print("")
            print("ERROR: Failure reading test list: " + test_list_file)
            print("       (errno {0}): {1}".format(e.errno, e.strerror))
            print("")
            sys.exit(1)

    def build_test_list(self, test_group, run_small, run_medium, run_large, run_xlarge, nopass):
        """
        Recursively find the list of tests to run and store them in the object.
        Fills in self.tests and self.tests_not_started.

        @param test_group: Name of the test group of tests to run.
        @return:  none
        """
        if (self.terminated):
            return

        for root, dirs, files in os.walk(self.test_root_dir):
            if (root.endswith("Util")):
                continue

            for f in files:
                # Figure out if the current file under consideration is a test.
                is_test = False
                if (is_python_test_file(f)):
                    is_test = True
                if (is_runit_test_file(f)):
                    is_test = True
                if (not is_test):
                    continue

                is_small = False
                is_medium = False
                is_large = False
                is_xlarge = False
                is_nopass = False
                is_nofeature = False

                if "xlarge" in f:
                    is_xlarge = True
                elif "medium" in f:
                    is_medium = True
                elif "large" in f:
                    is_large = True
                else:
                    is_small = True

                if "NOPASS" in f:
                    is_nopass = True
                if "NOFEATURE" in f:
                    is_nofeature = True

                if is_small and not run_small:
                    continue
                if is_medium and not run_medium:
                    continue
                if is_large and not run_large:
                    continue
                if is_xlarge and not run_xlarge:
                    continue

                if is_nopass and not nopass:
                    # skip all NOPASS tests for regular runs but still count the number of NOPASS tests
                    self.nopass_counter += 1
                    continue
                if is_nofeature and not nopass:
                    # skip all NOFEATURE tests for regular runs but still count the number of NOFEATURE tests
                    self.nofeature_counter += 1
                    continue
                if nopass and not is_nopass and not is_nofeature:
                    # if g_nopass flag is set, then ONLY run the NOPASS and NOFEATURE tests (skip all other tests)
                    continue

                if test_group is not None:
                    test_short_dir = self._calc_test_short_dir(os.path.join(root, f))
                    if (test_group.lower() not in test_short_dir) and test_group.lower() not in f:
                        continue

                self.add_test(os.path.join(root, f))

    def add_test(self, test_path):
        """
        Add one test to the list of tests to run.

        @param test_path: File system path to the test.
        @return: none
        """
        abs_test_path = os.path.abspath(test_path)
        abs_test_dir = os.path.dirname(abs_test_path)
        test_file = os.path.basename(abs_test_path)

        if (not os.path.exists(abs_test_path)):
            print("")
            print("ERROR: Test does not exist: " + abs_test_path)
            print("")
            sys.exit(1)

        test_short_dir = self._calc_test_short_dir(test_path)

        test = Test(abs_test_dir, test_short_dir, test_file, self.output_dir)
        self.tests.append(test)
        self.tests_not_started.append(test)

    def start_clouds(self):
        """
        Start all H2O clouds.

        @return: none
        """
        if (self.terminated):
            return

        if (self.use_cloud):
            return

        print("")
        print("Starting clouds...")
        print("")

        for cloud in self.clouds:
            if (self.terminated):
                return
            cloud.start()

        print("")
        print("Waiting for H2O nodes to come up...")
        print("")

        for cloud in self.clouds:
            if (self.terminated):
                return
            cloud.wait_for_cloud_to_be_up()

    def run_tests(self, nopass):
        """
        Run all tests.

        @return: none
        """
        if (self.terminated):
            return

        if (self._have_some_r_tests()):
            self._log("")
            self._log("Setting up R H2O package...")
            out_file_name = os.path.join(self.output_dir, "runnerSetupPackage.out.txt")
            out = open(out_file_name, "w")

            runner_setup_package_r = None
            if (True):
                possible_utils_parent_dir = self.test_root_dir
                while (True):
                    possible_utils_dir = os.path.join(possible_utils_parent_dir,
                                                      os.path.join("h2o-r",
                                                                   os.path.join("tests", "Utils")))
                    possible_runner_setup_package_r = os.path.join(possible_utils_dir, "runnerSetupPackage.R")
                    if (os.path.exists(possible_runner_setup_package_r)):
                        runner_setup_package_r = possible_runner_setup_package_r
                        break

                    next_possible_utils_parent_dir = os.path.dirname(possible_utils_parent_dir)
                    if (next_possible_utils_parent_dir == possible_utils_parent_dir):
                        break

                    possible_utils_parent_dir = next_possible_utils_parent_dir

            if (runner_setup_package_r is None):
                print("")
                print("ERROR: runnerSetupPackage.R not found.")
                print("")
                sys.exit(1)

            cmd = ["R",
                   "--quiet",
                   "-f",
                   runner_setup_package_r]
            if self.path_to_tar is not None:
                print "Using R TAR located at: " + self.path_to_tar
                cmd += ["--args", self.path_to_tar]
            child = subprocess.Popen(args=cmd,
                                     stdout=out,
                                     stderr=subprocess.STDOUT)
            rv = child.wait()
            if (self.terminated):
                return
            if (rv != 0):
                print("")
                print("ERROR: " + runner_setup_package_r + " failed.")
                print("       (See " + out_file_name + ")")
                print("")
                sys.exit(1)
            out.close()

        elif self._have_some_py_tests() and self.path_to_whl is not None:
            # basically only do this if we have a whl to install
            self._log("")
            self._log("Setting up Python H2O package...")
            out_file_name = os.path.join(self.output_dir, "pythonSetup.out.txt")
            out = open(out_file_name, "w")

            cmd = ["pip", "install", self.path_to_whl, "--force-reinstall"]
            child = subprocess.Popen(args=cmd,
                                     stdout=out,
                                     stderr=subprocess.STDOUT)
            rv = child.wait()
            if (self.terminated):
                return
            if (rv != 0):
                print("")
                print("ERROR: Python setup failed.")
                print("       (See " + out_file_name + ")")
                print("")
                sys.exit(1)
            out.close()

        num_tests = len(self.tests)
        num_nodes = self.num_clouds * self.nodes_per_cloud
        self._log("")
        if (self.use_client):
            client_message = " (+ client mode)"
        else:
            client_message = ""
        if (self.use_cloud):
            self._log("Starting {} tests...".format(num_tests))
        elif (self.use_cloud2):
            self._log("Starting {} tests on {} clouds...".format(num_tests, len(self.clouds)))
        else:
            self._log("Starting {} tests on {} clouds with {} total H2O worker nodes{}...".format(num_tests,
                                                                                                  self.num_clouds,
                                                                                                  num_nodes,
                                                                                                  client_message))
        self._log("")

        # Start the first n tests, where n is the lesser of the total number of tests and the total number of clouds.
        start_count = min(len(self.tests_not_started), len(self.clouds), 30)
        if (g_use_cloud2):
            start_count = min(start_count, 75)  # only open up 30 processes locally
        for i in range(start_count):
            cloud = self.clouds[i]
            ip = cloud.get_ip()
            port = cloud.get_port()
            self._start_next_test_on_ip_port(ip, port)

        # As each test finishes, send a new one to the cloud that just freed up.
        while (len(self.tests_not_started) > 0):
            if (self.terminated):
                return
            completed_test = self._wait_for_one_test_to_complete()
            if (self.terminated):
                return
            self._report_test_result(completed_test, nopass)
            ip_of_completed_test = completed_test.get_ip()
            port_of_completed_test = completed_test.get_port()
            self._start_next_test_on_ip_port(ip_of_completed_test, port_of_completed_test)

        # Wait for remaining running tests to complete.
        while (len(self.tests_running) > 0):
            if (self.terminated):
                return
            completed_test = self._wait_for_one_test_to_complete()
            if (self.terminated):
                return
            self._report_test_result(completed_test, nopass)

    def stop_clouds(self):
        """
        Stop all H2O clouds.

        @return: none
        """
        if (self.terminated):
            return

        if (self.use_cloud or self.use_cloud2):
            print("")
            print("All tests completed...")
            print("")
            return

        print("")
        print("All tests completed; tearing down clouds...")
        print("")
        for cloud in self.clouds:
            cloud.stop()

    def report_summary(self, nopass):
        """
        Report some summary information when the tests have finished running.

        @return: none
        """
        passed = 0
        nopass_but_tolerate = 0
        nofeature_but_tolerate = 0
        failed = 0
        notrun = 0
        total = 0
        true_fail_list = []
        terminated_list = []
        for test in self.tests:
            if (test.get_passed()):
                passed += 1
            else:
                if (test.get_nopass(nopass)):
                    nopass_but_tolerate += 1

                if (test.get_nofeature(nopass)):
                    nofeature_but_tolerate += 1

                if (test.get_completed()):
                    failed += 1
                    if (not test.get_nopass(nopass) and not test.get_nofeature(nopass)):
                        true_fail_list.append(test.get_test_name())
                else:
                    notrun += 1

                if (test.get_terminated()):
                    terminated_list.append(test.get_test_name())
            total += 1

        if ((passed + nopass_but_tolerate + nofeature_but_tolerate) == total):
            self.regression_passed = True
        else:
            self.regression_passed = False

        end_seconds = time.time()
        delta_seconds = end_seconds - self.start_seconds
        run = total - notrun
        self._log("")
        self._log("----------------------------------------------------------------------")
        self._log("")
        self._log("SUMMARY OF RESULTS")
        self._log("")
        self._log("----------------------------------------------------------------------")
        self._log("")
        self._log("Total tests:             " + str(total))
        self._log("Passed:                  " + str(passed))
        self._log("Did not pass:            " + str(failed))
        self._log("Did not complete:        " + str(notrun))
        self._log("Tolerated NOPASS:        " + str(nopass_but_tolerate))
        self._log("Tolerated NOFEATURE:     " + str(nofeature_but_tolerate))
        self._log("NOPASS tests skipped:    " + str(self.nopass_counter))
        self._log("NOFEATURE tests skipped: " + str(self.nofeature_counter))
        self._log("")
        self._log("Total time:              %.2f sec" % delta_seconds)
        if (run > 0):
            self._log("Time/completed test:     %.2f sec" % (delta_seconds / run))
        else:
            self._log("Time/completed test:     N/A")
        self._log("")
        if (len(true_fail_list) > 0):
            self._log("True fail list:          " + ", ".join(true_fail_list))
        if (len(terminated_list) > 0):
            self._log("Terminated list:         " + ", ".join(terminated_list))
        self._log("")

    def terminate(self):
        """
        Terminate all running clouds.  (Due to a signal.)

        @return: none
        """
        self.terminated = True

        for test in self.tests:
            test.cancel()

        for test in self.tests:
            test.terminate_if_started()

        for cloud in self.clouds:
            cloud.terminate()

    def get_regression_passed(self):
        """
        Return whether the overall regression passed or not.

        @return: true if the exit value should be 0, false otherwise.
        """
        return self.regression_passed

    # --------------------------------------------------------------------
    # Private methods below this line.
    # --------------------------------------------------------------------

    def _calc_test_short_dir(self, test_path):
        """
        Calculate directory of test relative to test_root_dir.

        @param test_path: Path to test file.
        @return: test_short_dir, relative directory containing test (relative to test_root_dir).
        """
        abs_test_root_dir = os.path.abspath(self.test_root_dir)
        abs_test_path = os.path.abspath(test_path)
        abs_test_dir = os.path.dirname(abs_test_path)

        test_short_dir = abs_test_dir

        # Look to elide longest prefix first.
        prefix = os.path.join(abs_test_root_dir, "")
        if (test_short_dir.startswith(prefix)):
            test_short_dir = test_short_dir.replace(prefix, "", 1)

        prefix = abs_test_root_dir
        if (test_short_dir.startswith(prefix)):
            test_short_dir = test_short_dir.replace(prefix, "", 1)

        return test_short_dir

    def _have_some_r_tests(self):
        """
        Do we have any R tests to run at all?
        (There might be tests of a different language to run, even if there are no R tests.)
        """
        for test in self.tests:
            test_name = test.get_test_name()
            if (is_runit_test_file(test_name)):
                return True

        return False

    def _have_some_py_tests(self):
        """
        dumbass check for pyuntis
        """
        for test in self.tests:
            test_name = test.get_test_name()
            if (is_python_test_file(test_name)):
                return True

        return False

    def _create_failed_output_dir(self):
        try:
            os.makedirs(self.failed_output_dir)
        except OSError as e:
            print("")
            print("mkdir failed (errno {0}): {1}".format(e.errno, e.strerror))
            print("    " + self.failed_output_dir)
            print("")
            print("(try adding --wipe)")
            print("")
            sys.exit(1)

    def _create_output_dir(self):
        try:
            os.makedirs(self.output_dir)
        except OSError as e:
            print("")
            print("mkdir failed (errno {0}): {1}".format(e.errno, e.strerror))
            print("    " + self.output_dir)
            print("")
            print("(try adding --wipe)")
            print("")
            sys.exit(1)

    def _create_testreport_dir(self):
        try:
            if not os.path.exists(self.testreport_dir):
                os.makedirs(self.testreport_dir)
        except OSError as e:
            print("")
            print("mkdir failed (errno {0}): {1}".format(e.errno, e.strerror))
            print("    " + self.testreport_dir)
            print("")
            sys.exit(1)

    def _start_next_test_on_ip_port(self, ip, port):
        test = self.tests_not_started.pop(0)
        self.tests_running.append(test)
        test.start(ip, port)

    def _wait_for_one_test_to_complete(self):
        while (True):
            for test in self.tests_running:
                if (self.terminated):
                    return None
                if (test.is_completed()):
                    self.tests_running.remove(test)
                    return test
            if (self.terminated):
                return
            time.sleep(1)

    def _report_test_result(self, test, nopass):
        port = test.get_port()
        now = time.time()
        duration = now - test.start_seconds
        test_name = test.get_test_name()
        if (test.get_passed()):
            s = "PASS      %d %4ds %-60s" % (port, duration, test_name)
            self._log(s)
            if self.produce_unit_reports:
                self._report_xunit_result("r_suite", test_name, duration, False)
        else:
            s = "     FAIL %d %4ds %-60s %s  %s" % \
                (port, duration, test.get_test_name(), test.get_output_dir_file_name(), test.get_seed_used())
            self._log(s)
            f = self._get_failed_filehandle_for_appending()
            f.write(test.get_test_dir_file_name() + "\n")
            f.close()
            # Report junit
            if self.produce_unit_reports:
                if not test.get_nopass(nopass):
                    self._report_xunit_result("r_suite", test_name, duration, False, "TestFailure", "Test failed", "See {}".format(test.get_output_dir_file_name()))
                else:
                    self._report_xunit_result("r_suite", test_name, duration, True)
            # Copy failed test output into directory failed
            if not test.get_nopass(nopass) and not test.get_nofeature(nopass):
                shutil.copy(test.get_output_dir_file_name(), self.failed_output_dir)

    # XSD schema for xunit reports is here; http://windyroad.com.au/dl/Open%20Source/JUnit.xsd
    def _report_xunit_result(self, testsuiteName, testcaseName, testcaseRuntime, skipped=False, failureType=None, failureMessage=None, failureDescription=None):
        errors = 0
        failures = 1 if failureType else 0
        skip = 1 if skipped else 0
        failure = "" if not failureType else """"<failure type="{}" message="{}">{}</failure>""".format(failureType, failureMessage, failureDescription)

	xmlReport= """<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="{testsuiteName}" tests="1" errors="{errors}" failures="{failures}" skip="{skip}">
  <testcase classname="{testcaseClassName}" name="{testcaseName}" time="{testcaseRuntime}">
  {failure}
  </testcase>
</testsuite>
""".format(testsuiteName=testsuiteName, testcaseClassName=testcaseName, testcaseName=testcaseName,
        testcaseRuntime=testcaseRuntime, failure=failure,
        errors=errors, failures=failures, skip=skip)
        self._save_xunit_report(testsuiteName, testcaseName, xmlReport)

    def _save_xunit_report(self, testsuite, testcase, report):
        f = self._get_testreport_filehandle(testsuite, testcase)
        f.write(report)
        f.close()

    def _log(self, s):
        f = self._get_summary_filehandle_for_appending()
        print(s)
        sys.stdout.flush()
        f.write(s + "\n")
        f.close()

    def _get_summary_filehandle_for_appending(self):
        summary_file_name = os.path.join(self.output_dir, "summary.txt")
        f = open(summary_file_name, "a+")
        return f<commit_msg>Fix capitalization and whitespace issues for PEP-8.<commit_after>#!/usr/bin/python

import sys
import os
import shutil
import signal
import time
import random
import getpass
import re
import subprocess
import ConfigParser


def is_python_test_file(file_name):
    """
    Return True if file_name matches a regexp for a python test.  False otherwise.
    """

    if (file_name == "test_config.py"):
        return False

    if re.match("^pyunit.*\.py$", file_name):
        return True

    if (re.match("^test.*\.py$", file_name)):
        return True

    return False


def is_python_file(file_name):
    """
    Return True if file_name matches a regexp for a python program in general.  False otherwise.

    This is a separate function because it's useful to have the scan-for-test operation in
    build_test_list() be separated from running the test.

    That allows us to run things explictly named using the --test option.  Such as:
        run.py --wipeall --numclouds 1 --test generate_rest_api_docs.py
    """

    if (file_name == "test_config.py"):
        return False

    if (re.match("^.*\.py$", file_name)):
        return True

    return False


def is_javascript_test_file(file_name):
    """
    Return True if file_name matches a regexp for a javascript test.  False otherwise.
    """

    if (re.match("^.*test.*\.js$", file_name)):
        return True

    return False


def is_runit_test_file(file_name):
    """
    Return True if file_name matches a regexp for a R test.  False otherwise.
    """

    if (file_name == "h2o-runit.R"):
        return False

    if (re.match("^runit.*\.[rR]$", file_name)):
        return True

    return False


class H2OUseCloudNode:
    """
    A class representing one node in an H2O cloud which was specified by the user.
    Don't try to build or tear down this kind of node.

    use_ip: The given ip of the cloud.
    use_port: The given port of the cloud.
    """

    def __init__(self, use_ip, use_port):
        self.use_ip = use_ip
        self.use_port = use_port

    def start(self):
        pass

    def stop(self):
        pass

    def terminate(self):
        pass

    def get_ip(self):
        return self.use_ip

    def get_port(self):
        return self.use_port


class H2OUseCloud:
    """
    A class representing an H2O clouds which was specified by the user.
    Don't try to build or tear down this kind of cloud.
    """

    def __init__(self, cloud_num, use_ip, use_port):
        self.cloud_num = cloud_num
        self.use_ip = use_ip
        self.use_port = use_port

        self.nodes = []
        node = H2OUseCloudNode(self.use_ip, self.use_port)
        self.nodes.append(node)

    def start(self):
        pass

    def wait_for_cloud_to_be_up(self):
        pass

    def stop(self):
        pass

    def terminate(self):
        pass

    def get_ip(self):
        node = self.nodes[0]
        return node.get_ip()

    def get_port(self):
        node = self.nodes[0]
        return node.get_port()


class H2OCloudNode:
    """
    A class representing one node in an H2O cloud.
    Note that the base_port is only a request for H2O.
    H2O may choose to ignore our request and pick any port it likes.
    So we have to scrape the real port number from stdout as part of cloud startup.

    port: The actual port chosen at run time.
    pid: The process id of the node.
    output_file_name: Where stdout and stderr go.  They are merged.
    child: subprocess.Popen object.
    terminated: Only from a signal.  Not normal shutdown.
    """

    def __init__(self, is_client, cloud_num, nodes_per_cloud, node_num, cloud_name, h2o_jar, ip, base_port,
                 xmx, output_dir):
        """
        Create a node in a cloud.

        @param is_client: Whether this node is an H2O client node (vs a worker node) or not.
        @param cloud_num: Dense 0-based cloud index number.
        @param nodes_per_cloud: How many H2O java instances are in a cloud.  Clouds are symmetric.
        @param node_num: This node's dense 0-based node index number.
        @param cloud_name: The H2O -name command-line argument.
        @param h2o_jar: Path to H2O jar file.
        @param base_port: The starting port number we are trying to get our nodes to listen on.
        @param xmx: Java memory parameter.
        @param output_dir: The directory where we can create an output file for this process.
        @return: The node object.
        """
        self.is_client = is_client
        self.cloud_num = cloud_num
        self.nodes_per_cloud = nodes_per_cloud
        self.node_num = node_num
        self.cloud_name = cloud_name
        self.h2o_jar = h2o_jar
        self.ip = ip
        self.base_port = base_port
        self.xmx = xmx
        self.output_dir = output_dir

        self.port = -1
        self.pid = -1
        self.output_file_name = ""
        self.child = None
        self.terminated = False

        # Choose my base port number here.  All math is done here.  Every node has the same
        # base_port and calculates it's own my_base_port.
        ports_per_node = 2
        self.my_base_port = \
            self.base_port + \
            (self.cloud_num * self.nodes_per_cloud * ports_per_node) + \
            (self.node_num * ports_per_node)

    def start(self):
        """
        Start one node of H2O.
        (Stash away the self.child and self.pid internally here.)

        @return: none
        """

        # there is no hdfs currently in ec2, except s3n/hdfs
        # the core-site.xml provides s3n info
        # it's possible that we can just always hardware the hdfs version
        # to match the cdh3 cluster we're hardwiring tests to
        # i.e. it won't make s3n/s3 break on ec2

        if (self.is_client):
            main_class = "water.H2OClientApp"
        else:
            main_class = "water.H2OApp"
        if "JAVA_HOME" in os.environ:
            java = os.environ["JAVA_HOME"] + "/bin/java"
        else:
            java = "java"
        cmd = [java,
               # "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005",
               "-Xmx" + self.xmx,
               "-ea",
               "-cp", self.h2o_jar,
               main_class,
               "-name", self.cloud_name,
               "-baseport", str(self.my_base_port)]

        # Add S3N credentials to cmd if they exist.
        # ec2_hdfs_config_file_name = os.path.expanduser("~/.ec2/core-site.xml")
        # if (os.path.exists(ec2_hdfs_config_file_name)):
        #     cmd.append("-hdfs_config")
        #     cmd.append(ec2_hdfs_config_file_name)

        self.output_file_name = \
            os.path.join(self.output_dir, "java_" + str(self.cloud_num) + "_" + str(self.node_num) + ".out.txt")
        f = open(self.output_file_name, "w")

        if g_convenient:
            cwd = os.getcwd()
            here = os.path.abspath(os.path.dirname(__file__))
            there = os.path.abspath(os.path.join(here, ".."))
            os.chdir(there)
            self.child = subprocess.Popen(args=cmd,
                                          stdout=f,
                                          stderr=subprocess.STDOUT,
                                          cwd=there)
            os.chdir(cwd)
        else:
            self.child = subprocess.Popen(args=cmd,
                                          stdout=f,
                                          stderr=subprocess.STDOUT,
                                          cwd=self.output_dir)

        self.pid = self.child.pid
        print("+ CMD: " + ' '.join(cmd))

    def scrape_port_from_stdout(self):
        """
        Look at the stdout log and figure out which port the JVM chose.
        Write this to self.port.
        This call is blocking.
        Exit if this fails.

        @return: none
        """
        retries = 30
        while (retries > 0):
            if (self.terminated):
                return
            f = open(self.output_file_name, "r")
            s = f.readline()
            while (len(s) > 0):
                if (self.terminated):
                    return
                match_groups = re.search(r"Listening for HTTP and REST traffic on  http://(\S+):(\d+)", s)
                if (match_groups is not None):
                    port = match_groups.group(2)
                    if (port is not None):
                        self.port = port
                        f.close()
                        print("H2O Cloud {} Node {} started with output file {}".format(self.cloud_num,
                                                                                        self.node_num,
                                                                                        self.output_file_name))
                        return

                s = f.readline()

            f.close()
            retries -= 1
            if (self.terminated):
                return
            time.sleep(1)

        print("")
        print("ERROR: Too many retries starting cloud.")
        print("")
        sys.exit(1)

    def scrape_cloudsize_from_stdout(self, nodes_per_cloud):
        """
        Look at the stdout log and wait until the cloud of proper size is formed.
        This call is blocking.
        Exit if this fails.

        @return: none
        """
        retries = 60
        while (retries > 0):
            if (self.terminated):
                return
            f = open(self.output_file_name, "r")
            s = f.readline()
            while (len(s) > 0):
                if (self.terminated):
                    return
                match_groups = re.search(r"Cloud of size (\d+) formed", s)
                if (match_groups is not None):
                    size = match_groups.group(1)
                    if (size is not None):
                        size = int(size)
                        if (size == nodes_per_cloud):
                            f.close()
                            return

                s = f.readline()

            f.close()
            retries -= 1
            if (self.terminated):
                return
            time.sleep(1)

        print("")
        print("ERROR: Too many retries starting cloud.")
        print("")
        sys.exit(1)

    def stop(self):
        """
        Normal node shutdown.
        Ignore failures for now.

        @return: none
        """
        if (self.pid > 0):
            print("Killing JVM with PID {}".format(self.pid))
            try:
                self.child.terminate()
                self.child.wait()
            except OSError:
                pass
            self.pid = -1

    def terminate(self):
        """
        Terminate a running node.  (Due to a signal.)

        @return: none
        """
        self.terminated = True
        self.stop()

    def get_ip(self):
        """ Return the ip address this node is really listening on. """
        return self.ip

    def get_port(self):
        """ Return the port this node is really listening on. """
        return self.port

    def __str__(self):
        s = ""
        s += "    node {}\n".format(self.node_num)
        s += "        xmx:          {}\n".format(self.xmx)
        s += "        my_base_port: {}\n".format(self.my_base_port)
        s += "        port:         {}\n".format(self.port)
        s += "        pid:          {}\n".format(self.pid)
        return s


class H2OCloud:
    """
    A class representing one of the H2O clouds.
    """

    def __init__(self, cloud_num, use_client, nodes_per_cloud, h2o_jar, base_port, xmx, output_dir):
        """
        Create a cloud.
        See node definition above for argument descriptions.

        @return: The cloud object.
        """
        self.use_client = use_client
        self.cloud_num = cloud_num
        self.nodes_per_cloud = nodes_per_cloud
        self.h2o_jar = h2o_jar
        self.base_port = base_port
        self.xmx = xmx
        self.output_dir = output_dir

        # Randomly choose a seven digit cloud number.
        n = random.randint(1000000, 9999999)
        user = getpass.getuser()
        user = ''.join(user.split())

        self.cloud_name = "H2O_runit_{}_{}".format(user, n)
        self.nodes = []
        self.client_nodes = []
        self.jobs_run = 0

        if (use_client):
            actual_nodes_per_cloud = self.nodes_per_cloud + 1
        else:
            actual_nodes_per_cloud = self.nodes_per_cloud

        for node_num in range(actual_nodes_per_cloud):
            is_client = False
            if (use_client):
                if (node_num == (actual_nodes_per_cloud - 1)):
                    is_client = True
            node = H2OCloudNode(is_client,
                                self.cloud_num, actual_nodes_per_cloud, node_num,
                                self.cloud_name,
                                self.h2o_jar,
                                "127.0.0.1", self.base_port,
                                self.xmx, self.output_dir)
            if (is_client):
                self.client_nodes.append(node)
            else:
                self.nodes.append(node)

    def start(self):
        """
        Start H2O cloud.
        The cloud is not up until wait_for_cloud_to_be_up() is called and returns.

        @return: none
        """
        for node in self.nodes:
            node.start()

        for node in self.client_nodes:
            node.start()

    def wait_for_cloud_to_be_up(self):
        """
        Blocking call ensuring the cloud is available.

        @return: none
        """
        self._scrape_port_from_stdout()
        self._scrape_cloudsize_from_stdout()

    def stop(self):
        """
        Normal cloud shutdown.

        @return: none
        """
        for node in self.nodes:
            node.stop()

        for node in self.client_nodes:
            node.stop()

    def terminate(self):
        """
        Terminate a running cloud.  (Due to a signal.)

        @return: none
        """
        for node in self.client_nodes:
            node.terminate()

        for node in self.nodes:
            node.terminate()

    def get_ip(self):
        """ Return an ip to use to talk to this cloud. """
        if (len(self.client_nodes) > 0):
            node = self.client_nodes[0]
        else:
            node = self.nodes[0]
        return node.get_ip()

    def get_port(self):
        """ Return a port to use to talk to this cloud. """
        if (len(self.client_nodes) > 0):
            node = self.client_nodes[0]
        else:
            node = self.nodes[0]
        return node.get_port()

    def _scrape_port_from_stdout(self):
        for node in self.nodes:
            node.scrape_port_from_stdout()
        for node in self.client_nodes:
            node.scrape_port_from_stdout()

    def _scrape_cloudsize_from_stdout(self):
        for node in self.nodes:
            node.scrape_cloudsize_from_stdout(self.nodes_per_cloud)
        for node in self.client_nodes:
            node.scrape_cloudsize_from_stdout(self.nodes_per_cloud)

    def __str__(self):
        s = ""
        s += "cloud {}\n".format(self.cloud_num)
        s += "    name:     {}\n".format(self.cloud_name)
        s += "    jobs_run: {}\n".format(self.jobs_run)
        for node in self.nodes:
            s += str(node)
        for node in self.client_nodes:
            s += str(node)
        return s


class Test:
    """
    A class representing one Test.

    cancelled: Don't start this test.
    terminated: Test killed due to signal.
    returncode: Exit code of child.
    pid: Process id of the test.
    ip: IP of cloud to run test.
    port: Port of cloud to run test.
    child: subprocess.Popen object.
    """

    @staticmethod
    def test_did_not_complete():
        """
        returncode marker to know if the test ran or not.
        """
        return -9999999

    def __init__(self, test_dir, test_short_dir, test_name, output_dir):
        """
        Create a Test.

        @param test_dir: Full absolute path to the test directory.
        @param test_short_dir: Path from h2o/R/tests to the test directory.
        @param test_name: Test filename with the directory removed.
        @param output_dir: The directory where we can create an output file for this process.
        @return: The test object.
        """
        self.test_dir = test_dir
        self.test_short_dir = test_short_dir
        self.test_name = test_name
        self.output_dir = output_dir
        self.output_file_name = ""

        self.cancelled = False
        self.terminated = False
        self.returncode = Test.test_did_not_complete()
        self.start_seconds = -1
        self.pid = -1
        self.ip = None
        self.port = -1
        self.child = None

    def start(self, ip, port):
        """
        Start the test in a non-blocking fashion.

        @param ip: IP address of cloud to run on.
        @param port: Port of cloud to run on.
        @return: none
        """
        if (self.cancelled or self.terminated):
            return

        self.start_seconds = time.time()
        self.ip = ip
        self.port = port

        if (is_python_test_file(self.test_name)):
            cmd = ["python",
                   self.test_name,
                   "--usecloud",
                   self.ip + ":" + str(self.port)]
        elif (is_python_file(self.test_name)):
            cmd = ["python",
                   self.test_name,
                   "--usecloud",
                   self.ip + ":" + str(self.port)]
        elif (is_runit_test_file(self.test_name)):
            cmd = ["R",
                   "-f",
                   self.test_name,
                   "--args",
                   self.ip + ":" + str(self.port)]
        elif (is_javascript_test_file(self.test_name)):
            cmd = ["phantomjs",
                   self.test_name,
                   self.ip + ":" + str(self.port)]
        else:
            print("")
            print("ERROR: Test runner failure with test: " + self.test_name)
            print("")
            sys.exit(1)

        test_short_dir_with_no_slashes = re.sub(r'[\\/]', "_", self.test_short_dir)
        if (len(test_short_dir_with_no_slashes) > 0):
            test_short_dir_with_no_slashes += "_"
        self.output_file_name = \
            os.path.join(self.output_dir, test_short_dir_with_no_slashes + self.test_name + ".out.txt")
        f = open(self.output_file_name, "w")
        self.child = subprocess.Popen(args=cmd,
                                      stdout=f,
                                      stderr=subprocess.STDOUT,
                                      cwd=self.test_dir)
        self.pid = self.child.pid
        # print("+ CMD: " + ' '.join(cmd))

    def is_completed(self):
        """
        Check if test has completed.

        This has side effects and MUST be called for the normal test queueing to work.
        Specifically, child.poll().

        @return: True if the test completed, False otherwise.
        """
        child = self.child
        if (child is None):
            return False
        child.poll()
        if (child.returncode is None):
            return False
        self.pid = -1
        self.returncode = child.returncode
        return True

    def cancel(self):
        """
        Mark this test as cancelled so it never tries to start.

        @return: none
        """
        if (self.pid <= 0):
            self.cancelled = True

    def terminate_if_started(self):
        """
        Terminate a running test.  (Due to a signal.)

        @return: none
        """
        if (self.pid > 0):
            self.terminate()

    def terminate(self):
        """
        Terminate a running test.  (Due to a signal.)

        @return: none
        """
        self.terminated = True
        if (self.pid > 0):
            print("Killing Test {} with PID {}".format(os.path.join(self.test_short_dir, self.test_name), self.pid))
            try:
                self.child.terminate()
            except OSError:
                pass
        self.pid = -1

    def get_test_dir_file_name(self):
        """
        @return: The full absolute path of this test.
        """
        return os.path.join(self.test_dir, self.test_name)

    def get_test_name(self):
        """
        @return: The file name (no directory) of this test.
        """
        return self.test_name

    def get_seed_used(self):
        """
        @return: The seed used by this test.
        """
        return self._scrape_output_for_seed()

    def get_ip(self):
        """
        @return: IP of the cloud where this test ran.
        """
        return self.ip

    def get_port(self):
        """
        @return: Integer port number of the cloud where this test ran.
        """
        return int(self.port)

    def get_passed(self):
        """
        @return: True if the test passed, False otherwise.
        """
        return (self.returncode == 0)

    def get_nopass(self, nopass):
        """
        Some tests are known not to fail and even if they don't pass we don't want
        to fail the overall regression PASS/FAIL status.

        @return: True if the test has been marked as NOPASS, False otherwise.
        """
        a = re.compile("NOPASS")
        return a.search(self.test_name) and not nopass

    def get_nofeature(self, nopass):
        """
        Some tests are known not to fail and even if they don't pass we don't want
        to fail the overall regression PASS/FAIL status.

        @return: True if the test has been marked as NOFEATURE, False otherwise.
        """
        a = re.compile("NOFEATURE")
        return a.search(self.test_name) and not nopass

    def get_completed(self):
        """
        @return: True if the test completed (pass or fail), False otherwise.
        """
        return (self.returncode > Test.test_did_not_complete())

    def get_terminated(self):
        """
        For a test to be terminated it must have started and had a PID.

        @return: True if the test was terminated, False otherwise.
        """
        return self.terminated

    def get_output_dir_file_name(self):
        """
        @return: Full path to the output file which you can paste to a terminal window.
        """
        return (os.path.join(self.output_dir, self.output_file_name))

    def _scrape_output_for_seed(self):
        """
        @return: The seed scraped from the outpul file.
        """
        res = ""
        with open(self.get_output_dir_file_name(), "r") as f:
            for line in f:
                if "SEED used" in line:
                    line = line.strip().split(' ')
                    res = line[-1]
                    break
        return res

    def __str__(self):
        s = ""
        s += "Test: {}/{}\n".format(self.test_dir, self.test_name)
        return s


class TestRunner:
    """
    A class for running tests.

    The tests list contains an object for every test.
    The tests_not_started list acts as a job queue.
    The tests_running list is polled for jobs that have finished.
    """

    def __init__(self,
                 test_root_dir,
                 use_cloud, use_cloud2, use_client, cloud_config, use_ip, use_port,
                 num_clouds, nodes_per_cloud, h2o_jar, base_port, xmx, output_dir,
                 failed_output_dir, path_to_tar, path_to_whl, produce_unit_reports, testreport_dir):
        """
        Create a runner.

        @param test_root_dir: h2o/R/tests directory.
        @param use_cloud: Use this one user-specified cloud.  Overrides num_clouds.
        @param use_cloud2: Use the cloud_config to define the list of H2O clouds.
        @param cloud_config: (if use_cloud2) the config file listing the H2O clouds.
        @param use_ip: (if use_cloud) IP of one cloud to use.
        @param use_port: (if use_cloud) Port of one cloud to use.
        @param num_clouds: Number of H2O clouds to start.
        @param nodes_per_cloud: Number of H2O nodes to start per cloud.
        @param h2o_jar: Path to H2O jar file to run.
        @param base_port: Base H2O port (e.g. 54321) to start choosing from.
        @param xmx: Java -Xmx parameter.
        @param output_dir: Directory for output files.
        @param failed_output_dir: Directory to copy failed test output.
        @param path_to_tar: NA
        @param path_to_whl: NA
        @param produce_unit_reports: if true then runner produce xUnit test reports for Jenkins
        @param testreport_dir: directory to put xUnit test reports for Jenkins (should follow build system conventions)
        @return: The runner object.
        """
        self.test_root_dir = test_root_dir

        self.use_cloud = use_cloud
        self.use_cloud2 = use_cloud2
        self.use_client = use_client

        # Valid if use_cloud is True
        self.use_ip = use_ip
        self.use_port = use_port

        # Valid if use_cloud is False
        self.num_clouds = num_clouds
        self.nodes_per_cloud = nodes_per_cloud
        self.h2o_jar = h2o_jar
        self.base_port = base_port
        self.output_dir = output_dir
        self.failed_output_dir = failed_output_dir
        self.produce_unit_reports = produce_unit_reports
        self.testreport_dir = testreport_dir

        self.start_seconds = time.time()
        self.terminated = False
        self.clouds = []
        self.tests = []
        self.tests_not_started = []
        self.tests_running = []
        self.regression_passed = False
        self._create_output_dir()
        self._create_failed_output_dir()
        if produce_unit_reports:
            self._create_testreport_dir()
        self.nopass_counter = 0
        self.nofeature_counter = 0
        self.path_to_tar = path_to_tar
        self.path_to_whl = path_to_whl

        if (use_cloud):
            node_num = 0
            cloud = H2OUseCloud(node_num, use_ip, use_port)
            self.clouds.append(cloud)
        elif (use_cloud2):
            clouds = TestRunner.read_config(cloud_config)
            node_num = 0
            for c in clouds:
                cloud = H2OUseCloud(node_num, c[0], c[1])
                self.clouds.append(cloud)
                node_num += 1
        else:
            for i in range(self.num_clouds):
                cloud = H2OCloud(i, self.use_client, self.nodes_per_cloud, h2o_jar, self.base_port, xmx,
                                 self.output_dir)
                self.clouds.append(cloud)

    @staticmethod
    def find_test(test_to_run):
        """
        Be nice and try to help find the test if possible.
        If the test is actually found without looking, then just use it.
        Otherwise, search from the script's down directory down.
        """
        if (os.path.exists(test_to_run)):
            abspath_test = os.path.abspath(test_to_run)
            return abspath_test

        for d, subdirs, files in os.walk(os.getcwd()):
            for f in files:
                if (f == test_to_run):
                    return os.path.join(d, f)

        # Not found, return the file, which will result in an error downstream when it can't be found.
        print("")
        print("ERROR: Test does not exist: " + test_to_run)
        print("")
        sys.exit(1)

    @staticmethod
    def read_config(config_file):
        clouds = []  # a list of lists. Inner lists have [node_num, ip, port]
        cfg = ConfigParser.RawConfigParser()
        cfg.read(config_file)
        for s in cfg.sections():
            items = cfg.items(s)
            cloud = [items[0][1], int(items[1][1])]
            clouds.append(cloud)
        return clouds

    def read_test_list_file(self, test_list_file):
        """
        Read in a test list file line by line.  Each line in the file is a test
        to add to the test run.

        @param test_list_file: Filesystem path to a file with a list of tests to run.
        @return: none
        """
        try:
            f = open(test_list_file, "r")
            s = f.readline()
            while (len(s) != 0):
                stripped = s.strip()
                if (len(stripped) == 0):
                    s = f.readline()
                    continue
                if (stripped.startswith("#")):
                    s = f.readline()
                    continue
                found_stripped = TestRunner.find_test(stripped)
                self.add_test(found_stripped)
                s = f.readline()
            f.close()
        except IOError as e:
            print("")
            print("ERROR: Failure reading test list: " + test_list_file)
            print("       (errno {0}): {1}".format(e.errno, e.strerror))
            print("")
            sys.exit(1)

    def build_test_list(self, test_group, run_small, run_medium, run_large, run_xlarge, nopass):
        """
        Recursively find the list of tests to run and store them in the object.
        Fills in self.tests and self.tests_not_started.

        @param test_group: Name of the test group of tests to run.
        @return:  none
        """
        if (self.terminated):
            return

        for root, dirs, files in os.walk(self.test_root_dir):
            if (root.endswith("Util")):
                continue

            for f in files:
                # Figure out if the current file under consideration is a test.
                is_test = False
                if (is_python_test_file(f)):
                    is_test = True
                if (is_runit_test_file(f)):
                    is_test = True
                if (not is_test):
                    continue

                is_small = False
                is_medium = False
                is_large = False
                is_xlarge = False
                is_nopass = False
                is_nofeature = False

                if "xlarge" in f:
                    is_xlarge = True
                elif "medium" in f:
                    is_medium = True
                elif "large" in f:
                    is_large = True
                else:
                    is_small = True

                if "NOPASS" in f:
                    is_nopass = True
                if "NOFEATURE" in f:
                    is_nofeature = True

                if is_small and not run_small:
                    continue
                if is_medium and not run_medium:
                    continue
                if is_large and not run_large:
                    continue
                if is_xlarge and not run_xlarge:
                    continue

                if is_nopass and not nopass:
                    # skip all NOPASS tests for regular runs but still count the number of NOPASS tests
                    self.nopass_counter += 1
                    continue
                if is_nofeature and not nopass:
                    # skip all NOFEATURE tests for regular runs but still count the number of NOFEATURE tests
                    self.nofeature_counter += 1
                    continue
                if nopass and not is_nopass and not is_nofeature:
                    # if g_nopass flag is set, then ONLY run the NOPASS and NOFEATURE tests (skip all other tests)
                    continue

                if test_group is not None:
                    test_short_dir = self._calc_test_short_dir(os.path.join(root, f))
                    if (test_group.lower() not in test_short_dir) and test_group.lower() not in f:
                        continue

                self.add_test(os.path.join(root, f))

    def add_test(self, test_path):
        """
        Add one test to the list of tests to run.

        @param test_path: File system path to the test.
        @return: none
        """
        abs_test_path = os.path.abspath(test_path)
        abs_test_dir = os.path.dirname(abs_test_path)
        test_file = os.path.basename(abs_test_path)

        if (not os.path.exists(abs_test_path)):
            print("")
            print("ERROR: Test does not exist: " + abs_test_path)
            print("")
            sys.exit(1)

        test_short_dir = self._calc_test_short_dir(test_path)

        test = Test(abs_test_dir, test_short_dir, test_file, self.output_dir)
        self.tests.append(test)
        self.tests_not_started.append(test)

    def start_clouds(self):
        """
        Start all H2O clouds.

        @return: none
        """
        if (self.terminated):
            return

        if (self.use_cloud):
            return

        print("")
        print("Starting clouds...")
        print("")

        for cloud in self.clouds:
            if (self.terminated):
                return
            cloud.start()

        print("")
        print("Waiting for H2O nodes to come up...")
        print("")

        for cloud in self.clouds:
            if (self.terminated):
                return
            cloud.wait_for_cloud_to_be_up()

    def run_tests(self, nopass):
        """
        Run all tests.

        @return: none
        """
        if (self.terminated):
            return

        if (self._have_some_r_tests()):
            self._log("")
            self._log("Setting up R H2O package...")
            out_file_name = os.path.join(self.output_dir, "runnerSetupPackage.out.txt")
            out = open(out_file_name, "w")

            runner_setup_package_r = None
            if (True):
                possible_utils_parent_dir = self.test_root_dir
                while (True):
                    possible_utils_dir = os.path.join(possible_utils_parent_dir,
                                                      os.path.join("h2o-r",
                                                                   os.path.join("tests", "Utils")))
                    possible_runner_setup_package_r = os.path.join(possible_utils_dir, "runnerSetupPackage.R")
                    if (os.path.exists(possible_runner_setup_package_r)):
                        runner_setup_package_r = possible_runner_setup_package_r
                        break

                    next_possible_utils_parent_dir = os.path.dirname(possible_utils_parent_dir)
                    if (next_possible_utils_parent_dir == possible_utils_parent_dir):
                        break

                    possible_utils_parent_dir = next_possible_utils_parent_dir

            if (runner_setup_package_r is None):
                print("")
                print("ERROR: runnerSetupPackage.R not found.")
                print("")
                sys.exit(1)

            cmd = ["R",
                   "--quiet",
                   "-f",
                   runner_setup_package_r]
            if self.path_to_tar is not None:
                print "Using R TAR located at: " + self.path_to_tar
                cmd += ["--args", self.path_to_tar]
            child = subprocess.Popen(args=cmd,
                                     stdout=out,
                                     stderr=subprocess.STDOUT)
            rv = child.wait()
            if (self.terminated):
                return
            if (rv != 0):
                print("")
                print("ERROR: " + runner_setup_package_r + " failed.")
                print("       (See " + out_file_name + ")")
                print("")
                sys.exit(1)
            out.close()

        elif self._have_some_py_tests() and self.path_to_whl is not None:
            # basically only do this if we have a whl to install
            self._log("")
            self._log("Setting up Python H2O package...")
            out_file_name = os.path.join(self.output_dir, "pythonSetup.out.txt")
            out = open(out_file_name, "w")

            cmd = ["pip", "install", self.path_to_whl, "--force-reinstall"]
            child = subprocess.Popen(args=cmd,
                                     stdout=out,
                                     stderr=subprocess.STDOUT)
            rv = child.wait()
            if (self.terminated):
                return
            if (rv != 0):
                print("")
                print("ERROR: Python setup failed.")
                print("       (See " + out_file_name + ")")
                print("")
                sys.exit(1)
            out.close()

        num_tests = len(self.tests)
        num_nodes = self.num_clouds * self.nodes_per_cloud
        self._log("")
        if (self.use_client):
            client_message = " (+ client mode)"
        else:
            client_message = ""
        if (self.use_cloud):
            self._log("Starting {} tests...".format(num_tests))
        elif (self.use_cloud2):
            self._log("Starting {} tests on {} clouds...".format(num_tests, len(self.clouds)))
        else:
            self._log("Starting {} tests on {} clouds with {} total H2O worker nodes{}...".format(num_tests,
                                                                                                  self.num_clouds,
                                                                                                  num_nodes,
                                                                                                  client_message))
        self._log("")

        # Start the first n tests, where n is the lesser of the total number of tests and the total number of clouds.
        start_count = min(len(self.tests_not_started), len(self.clouds), 30)
        if (g_use_cloud2):
            start_count = min(start_count, 75)  # only open up 30 processes locally
        for i in range(start_count):
            cloud = self.clouds[i]
            ip = cloud.get_ip()
            port = cloud.get_port()
            self._start_next_test_on_ip_port(ip, port)

        # As each test finishes, send a new one to the cloud that just freed up.
        while (len(self.tests_not_started) > 0):
            if (self.terminated):
                return
            completed_test = self._wait_for_one_test_to_complete()
            if (self.terminated):
                return
            self._report_test_result(completed_test, nopass)
            ip_of_completed_test = completed_test.get_ip()
            port_of_completed_test = completed_test.get_port()
            self._start_next_test_on_ip_port(ip_of_completed_test, port_of_completed_test)

        # Wait for remaining running tests to complete.
        while (len(self.tests_running) > 0):
            if (self.terminated):
                return
            completed_test = self._wait_for_one_test_to_complete()
            if (self.terminated):
                return
            self._report_test_result(completed_test, nopass)

    def stop_clouds(self):
        """
        Stop all H2O clouds.

        @return: none
        """
        if (self.terminated):
            return

        if (self.use_cloud or self.use_cloud2):
            print("")
            print("All tests completed...")
            print("")
            return

        print("")
        print("All tests completed; tearing down clouds...")
        print("")
        for cloud in self.clouds:
            cloud.stop()

    def report_summary(self, nopass):
        """
        Report some summary information when the tests have finished running.

        @return: none
        """
        passed = 0
        nopass_but_tolerate = 0
        nofeature_but_tolerate = 0
        failed = 0
        notrun = 0
        total = 0
        true_fail_list = []
        terminated_list = []
        for test in self.tests:
            if (test.get_passed()):
                passed += 1
            else:
                if (test.get_nopass(nopass)):
                    nopass_but_tolerate += 1

                if (test.get_nofeature(nopass)):
                    nofeature_but_tolerate += 1

                if (test.get_completed()):
                    failed += 1
                    if (not test.get_nopass(nopass) and not test.get_nofeature(nopass)):
                        true_fail_list.append(test.get_test_name())
                else:
                    notrun += 1

                if (test.get_terminated()):
                    terminated_list.append(test.get_test_name())
            total += 1

        if ((passed + nopass_but_tolerate + nofeature_but_tolerate) == total):
            self.regression_passed = True
        else:
            self.regression_passed = False

        end_seconds = time.time()
        delta_seconds = end_seconds - self.start_seconds
        run = total - notrun
        self._log("")
        self._log("----------------------------------------------------------------------")
        self._log("")
        self._log("SUMMARY OF RESULTS")
        self._log("")
        self._log("----------------------------------------------------------------------")
        self._log("")
        self._log("Total tests:             " + str(total))
        self._log("Passed:                  " + str(passed))
        self._log("Did not pass:            " + str(failed))
        self._log("Did not complete:        " + str(notrun))
        self._log("Tolerated NOPASS:        " + str(nopass_but_tolerate))
        self._log("Tolerated NOFEATURE:     " + str(nofeature_but_tolerate))
        self._log("NOPASS tests skipped:    " + str(self.nopass_counter))
        self._log("NOFEATURE tests skipped: " + str(self.nofeature_counter))
        self._log("")
        self._log("Total time:              %.2f sec" % delta_seconds)
        if (run > 0):
            self._log("Time/completed test:     %.2f sec" % (delta_seconds / run))
        else:
            self._log("Time/completed test:     N/A")
        self._log("")
        if (len(true_fail_list) > 0):
            self._log("True fail list:          " + ", ".join(true_fail_list))
        if (len(terminated_list) > 0):
            self._log("Terminated list:         " + ", ".join(terminated_list))
        self._log("")

    def terminate(self):
        """
        Terminate all running clouds.  (Due to a signal.)

        @return: none
        """
        self.terminated = True

        for test in self.tests:
            test.cancel()

        for test in self.tests:
            test.terminate_if_started()

        for cloud in self.clouds:
            cloud.terminate()

    def get_regression_passed(self):
        """
        Return whether the overall regression passed or not.

        @return: true if the exit value should be 0, false otherwise.
        """
        return self.regression_passed

    # --------------------------------------------------------------------
    # Private methods below this line.
    # --------------------------------------------------------------------

    def _calc_test_short_dir(self, test_path):
        """
        Calculate directory of test relative to test_root_dir.

        @param test_path: Path to test file.
        @return: test_short_dir, relative directory containing test (relative to test_root_dir).
        """
        abs_test_root_dir = os.path.abspath(self.test_root_dir)
        abs_test_path = os.path.abspath(test_path)
        abs_test_dir = os.path.dirname(abs_test_path)

        test_short_dir = abs_test_dir

        # Look to elide longest prefix first.
        prefix = os.path.join(abs_test_root_dir, "")
        if (test_short_dir.startswith(prefix)):
            test_short_dir = test_short_dir.replace(prefix, "", 1)

        prefix = abs_test_root_dir
        if (test_short_dir.startswith(prefix)):
            test_short_dir = test_short_dir.replace(prefix, "", 1)

        return test_short_dir

    def _have_some_r_tests(self):
        """
        Do we have any R tests to run at all?
        (There might be tests of a different language to run, even if there are no R tests.)
        """
        for test in self.tests:
            test_name = test.get_test_name()
            if (is_runit_test_file(test_name)):
                return True

        return False

    def _have_some_py_tests(self):
        """
        dumbass check for pyuntis
        """
        for test in self.tests:
            test_name = test.get_test_name()
            if (is_python_test_file(test_name)):
                return True

        return False

    def _create_failed_output_dir(self):
        try:
            os.makedirs(self.failed_output_dir)
        except OSError as e:
            print("")
            print("mkdir failed (errno {0}): {1}".format(e.errno, e.strerror))
            print("    " + self.failed_output_dir)
            print("")
            print("(try adding --wipe)")
            print("")
            sys.exit(1)

    def _create_output_dir(self):
        try:
            os.makedirs(self.output_dir)
        except OSError as e:
            print("")
            print("mkdir failed (errno {0}): {1}".format(e.errno, e.strerror))
            print("    " + self.output_dir)
            print("")
            print("(try adding --wipe)")
            print("")
            sys.exit(1)

    def _create_testreport_dir(self):
        try:
            if not os.path.exists(self.testreport_dir):
                os.makedirs(self.testreport_dir)
        except OSError as e:
            print("")
            print("mkdir failed (errno {0}): {1}".format(e.errno, e.strerror))
            print("    " + self.testreport_dir)
            print("")
            sys.exit(1)

    def _start_next_test_on_ip_port(self, ip, port):
        test = self.tests_not_started.pop(0)
        self.tests_running.append(test)
        test.start(ip, port)

    def _wait_for_one_test_to_complete(self):
        while (True):
            for test in self.tests_running:
                if (self.terminated):
                    return None
                if (test.is_completed()):
                    self.tests_running.remove(test)
                    return test
            if (self.terminated):
                return
            time.sleep(1)

    def _report_test_result(self, test, nopass):
        port = test.get_port()
        now = time.time()
        duration = now - test.start_seconds
        test_name = test.get_test_name()
        if (test.get_passed()):
            s = "PASS      %d %4ds %-60s" % (port, duration, test_name)
            self._log(s)
            if self.produce_unit_reports:
                self._report_xunit_result("r_suite", test_name, duration, False)
        else:
            s = "     FAIL %d %4ds %-60s %s  %s" % \
                (port, duration, test.get_test_name(), test.get_output_dir_file_name(), test.get_seed_used())
            self._log(s)
            f = self._get_failed_filehandle_for_appending()
            f.write(test.get_test_dir_file_name() + "\n")
            f.close()
            # Report junit
            if self.produce_unit_reports:
                if not test.get_nopass(nopass):
                    self._report_xunit_result("r_suite", test_name, duration, False, "TestFailure", "Test failed",
                                              "See {}".format(test.get_output_dir_file_name()))
                else:
                    self._report_xunit_result("r_suite", test_name, duration, True)
            # Copy failed test output into directory failed
            if not test.get_nopass(nopass) and not test.get_nofeature(nopass):
                shutil.copy(test.get_output_dir_file_name(), self.failed_output_dir)

    # XSD schema for xunit reports is here; http://windyroad.com.au/dl/Open%20Source/JUnit.xsd
    def _report_xunit_result(self, testsuite_name, testcase_name, testcase_runtime,
                             skipped=False, failure_type=None, failure_message=None, failure_description=None):
        errors = 0
        failures = 1 if failure_type else 0
        skip = 1 if skipped else 0
        failure = "" if not failure_type else """"<failure type="{}" message="{}">{}</failure>""" \
            .format(failure_type, failure_message, failure_description)

        xml_report = """<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="{testsuiteName}" tests="1" errors="{errors}" failures="{failures}" skip="{skip}">
  <testcase classname="{testcaseClassName}" name="{testcaseName}" time="{testcaseRuntime}">
  {failure}
  </testcase>
</testsuite>
""".format(testsuiteName=testsuite_name, testcaseClassName=testcase_name, testcaseName=testcase_name,
           testcaseRuntime=testcase_runtime, failure=failure,
           errors=errors, failures=failures, skip=skip)

        self._save_xunit_report(testsuite_name, testcase_name, xml_report)

    def _save_xunit_report(self, testsuite, testcase, report):
        f = self._get_testreport_filehandle(testsuite, testcase)
        f.write(report)
        f.close()

    def _log(self, s):
        f = self._get_summary_filehandle_for_appending()
        print(s)
        sys.stdout.flush()
        f.write(s + "\n")
        f.close()

    def _get_summary_filehandle_for_appending(self):
        summary_file_name = os.path.join(self.output_dir, "summary.txt")
        f = open(summary_file_name, "a+")
        return f
{'fraction_non_alphanumeric': 0.09927318756252425, 'fraction_numerical': 0.004910067168902227, 'mean_word_length': 1.5743154464708047, 'pattern_counts': {'":': 8, '<': 19, '<?xml version=': 2, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 2, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 90, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5954036', 'n_tokens_mistral': 28883, 'n_tokens_neox': 27484, 'n_words': 7396}