<commit_before>"""
An H2OConnection represents the latest active handle to a cloud. No more than a single
H2OConnection object will be active at any one time.
"""

import requests
import math
import re
import os
import sys
import time
import tempfile
import tabulate
import subprocess
import atexit
import pkg_resources
from two_dim_table import H2OTwoDimTable

__H2OCONN__ = None            # the single active connection to H2O cloud
__H2O_REST_API_VERSION__ = 3  # const for the version of the rest api


class H2OConnection(object):
  """
  H2OConnection is a class that represents a connection to the H2O cluster.
  It is specified by an IP address and a port number.

  Objects of type H2OConnection are not instantiated directly!

  This class contains static methods for performing the common REST methods
  GET, POST, and DELETE.
  """

  def __init__(self, ip="localhost", port=54321, size=1, start_h2o=False, enable_assertions=False,
               license=None, max_mem_size_GB=1, min_mem_size_GB=1, ice_root=None, strict_version_check=False):
    """
        Instantiate the package handle to the H2O cluster.
    :param ip: An IP address, default is "localhost"
    :param port: A port, default is 54321
    :param size: THe expected number of h2o instances (ignored if start_h2o is True)
    :param start_h2o: A boolean dictating whether this module should start the H2O jvm. An attempt is made anyways if _connect fails.
    :param enable_assertions: If start_h2o, pass `-ea` as a VM option.s
    :param license: If not None, is a path to a license file.
    :param max_mem_size_GB: Maximum heap size (jvm option Xmx) in gigabytes.
    :param min_mem_size_GB: Minimum heap size (jvm option Xms) in gigabytes.
    :param ice_root: A temporary directory (default location is determined by tempfile.mkdtemp()) to hold H2O log files.
    :return: None
    """
    port = as_int(port)
    if not (isinstance(port, int) and 0 <= port <= sys.maxint):
       raise ValueError("Port out of range, "+port)
    global __H2OCONN__
    self._ip = ip
    self._port = port
    self._session_id = None
    self._rest_version = __H2O_REST_API_VERSION__
    self._child = getattr(__H2OCONN__, "_child") if hasattr(__H2OCONN__, "_child") else None
    __H2OCONN__ = self
    if start_h2o:
      if not ice_root:
        ice_root = tempfile.mkdtemp()
      cld = self._start_local_h2o_jar(max_mem_size_GB, min_mem_size_GB, enable_assertions, license, ice_root)
    else:
      try:
        cld = self._connect(size)
      except:
        # try to start local jar or re-raise previous exception
        print
        print
        print "No instance found at ip and port: " + ip + ":" + str(port) + ". Trying to start local jar..."
        print
        print
        if os.path.exists(os.path.join(sys.prefix, "h2o_jar/h2o.jar")):
          if not ice_root:
            ice_root = tempfile.mkdtemp()
          cld = self._start_local_h2o_jar(max_mem_size_GB, min_mem_size_GB, enable_assertions, license, ice_root)
        else:
          print "No jar file found. Could not start local instance."
          raise

    # self._session_id = self.get_session_id()
    ncpus = sum([n['num_cpus'] for n in cld['nodes']])
    allowed_cpus = sum([n['cpus_allowed'] for n in cld['nodes']])
    mmax = sum([n['max_mem'] for n in cld['nodes']])
    cluster_health = all([n['healthy'] for n in cld['nodes']])
    cluster_info = [
      ["H2O cluster uptime: ", get_human_readable_time(cld["cloud_uptime_millis"])],
      ["H2O cluster version: ", cld["version"]],
      ["H2O cluster name: ", cld["cloud_name"]],
      ["H2O cluster total nodes: ", cld["cloud_size"]],
      ["H2O cluster total memory: ", get_human_readable_size(mmax)],
      ["H2O cluster total cores: ", str(ncpus)],
      ["H2O cluster allowed cores: ", str(allowed_cpus)],
      ["H2O cluster healthy: ", str(cluster_health)],
    ]
    print
    print tabulate.tabulate(cluster_info)
    print

    ver_h2o = cld['version']
    try:
      ver_pkg = pkg_resources.get_distribution("h2o").version
    except:
      ver_pkg = "UNKNOWN"

    if ver_h2o != ver_pkg:
      message = \
        "Version mismatch. H2O is version {0}, but the python package is version {1}.".format(ver_h2o, str(ver_pkg))
      if strict_version_check:
        raise EnvironmentError, message
      else:
        print "Warning: {0}".format(message)

  def _connect(self, size, max_retries=5, print_dots=False):
    """
    Does not actually "connect", instead simply tests that the cluster can be reached,
    is of a certain size, and is taking basic status commands.
    :param size: The number of H2O instances in the cloud.
    :return: The JSON response from a "stable" cluster.
    """
    max_retries = max_retries
    retries = 0
    while True:
      retries += 1
      if print_dots:
        self._print_dots(retries)
      try:
        cld = H2OConnection.get_json(url_suffix="Cloud")
        if not cld['cloud_healthy']:
          raise ValueError("Cluster reports unhealthy status", cld)
        if cld['cloud_size'] >= size and cld['consensus']:
          if print_dots: print " Connection sucessful!"
          return cld
      except EnvironmentError:
        pass
      # Cloud too small or voting in progress; sleep; try again
      time.sleep(0.1)
      if retries > max_retries:
        raise EnvironmentError("Max retries exceeded. Could not establish link to the H2O cloud @ " + str(self._ip) + ":" + str(self._port))

  def _print_dots(self, retries):
    sys.stdout.write("\rStarting H2O JVM and connecting: {}".format("." * retries))
    sys.stdout.flush()

  def _start_local_h2o_jar(self, mmax, mmin, ea, license, ice):
    command = H2OConnection._check_java()
    if license:
      if not os.path.exists(license):
        raise ValueError("License file not found (" + license + ")")

    if not ice:
      raise ValueError("`ice_root` must be specified")

    stdout = open(H2OConnection._tmp_file("stdout"), 'w')
    stderr = open(H2OConnection._tmp_file("stderr"), 'w')

    print "Using ice_root: " + ice
    print

    jar_file = os.path.join(sys.prefix, "h2o_jar/h2o.jar")

    jver = subprocess.check_output([command, "-version"], stderr=subprocess.STDOUT)

    print
    print "Java Version: " + jver
    print

    if "GNU libgcj" in jver:
      raise ValueError("Sorry, GNU Java is not supported for H2O.\n"+
                       "Please download the latest Java SE JDK 7 from the following URL:\n"+
                       "http://www.oracle.com/technetwork/java/javase/downloads/jdk7-downloads-1880260.html")

    if "Client VM" in jver:
      print "WARNING: "
      print "You have a 32-bit version of Java. H2O works best with 64-bit Java."
      print "Please download the latest Java SE JDK 7 from the following URL:"
      print "http://www.oracle.com/technetwork/java/javase/downloads/jdk7-downloads-1880260.html"
      print

    vm_opts = ["-Xms{}g".format(mmin), "-Xmx{}g".format(mmax)]
    if ea:
      vm_opts += ["-ea"]

    h2o_opts = ["-jar", jar_file,
                "-name", "H2O_started_from_python",
                "-ip", "127.0.0.1",
                "-port", "54321",
                "-ice_root", ice,
                ]
    if license:
      h2o_opts += ["-license", license]

    cmd = [command] + vm_opts + h2o_opts

    cwd = os.path.abspath(os.getcwd())
    self._child = subprocess.Popen(args=cmd, stdout=stdout, stderr=stderr, cwd=cwd)
    cld = self._connect(1, 30, True)
    return cld<commit_msg>changed default xmx/s behavior<commit_after>"""
An H2OConnection represents the latest active handle to a cloud. No more than a single
H2OConnection object will be active at any one time.
"""

import requests
import math
import re
import os
import sys
import time
import tempfile
import tabulate
import subprocess
import atexit
import pkg_resources
from two_dim_table import H2OTwoDimTable

__H2OCONN__ = None            # the single active connection to H2O cloud
__H2O_REST_API_VERSION__ = 3  # const for the version of the rest api


class H2OConnection(object):
  """
  H2OConnection is a class that represents a connection to the H2O cluster.
  It is specified by an IP address and a port number.

  Objects of type H2OConnection are not instantiated directly!

  This class contains static methods for performing the common REST methods
  GET, POST, and DELETE.
  """

  def __init__(self, ip="localhost", port=54321, size=1, start_h2o=False, enable_assertions=False,
               license=None, max_mem_size_GB=None, min_mem_size_GB=None, ice_root=None, strict_version_check=False):
    """
        Instantiate the package handle to the H2O cluster.
    :param ip: An IP address, default is "localhost"
    :param port: A port, default is 54321
    :param size: THe expected number of h2o instances (ignored if start_h2o is True)
    :param start_h2o: A boolean dictating whether this module should start the H2O jvm. An attempt is made anyways if _connect fails.
    :param enable_assertions: If start_h2o, pass `-ea` as a VM option.s
    :param license: If not None, is a path to a license file.
    :param max_mem_size_GB: Maximum heap size (jvm option Xmx) in gigabytes.
    :param min_mem_size_GB: Minimum heap size (jvm option Xms) in gigabytes.
    :param ice_root: A temporary directory (default location is determined by tempfile.mkdtemp()) to hold H2O log files.
    :return: None
    """
    port = as_int(port)
    if not (isinstance(port, int) and 0 <= port <= sys.maxint):
       raise ValueError("Port out of range, "+port)
    global __H2OCONN__
    self._ip = ip
    self._port = port
    self._session_id = None
    self._rest_version = __H2O_REST_API_VERSION__
    self._child = getattr(__H2OCONN__, "_child") if hasattr(__H2OCONN__, "_child") else None
    __H2OCONN__ = self
    if start_h2o:
      if not ice_root:
        ice_root = tempfile.mkdtemp()
      cld = self._start_local_h2o_jar(max_mem_size_GB, min_mem_size_GB, enable_assertions, license, ice_root)
    else:
      try:
        cld = self._connect(size)
      except:
        # try to start local jar or re-raise previous exception
        print
        print
        print "No instance found at ip and port: " + ip + ":" + str(port) + ". Trying to start local jar..."
        print
        print
        if os.path.exists(os.path.join(sys.prefix, "h2o_jar/h2o.jar")):
          if not ice_root:
            ice_root = tempfile.mkdtemp()
          cld = self._start_local_h2o_jar(max_mem_size_GB, min_mem_size_GB, enable_assertions, license, ice_root)
        else:
          print "No jar file found. Could not start local instance."
          raise

    # self._session_id = self.get_session_id()
    ncpus = sum([n['num_cpus'] for n in cld['nodes']])
    allowed_cpus = sum([n['cpus_allowed'] for n in cld['nodes']])
    mmax = sum([n['max_mem'] for n in cld['nodes']])
    cluster_health = all([n['healthy'] for n in cld['nodes']])
    cluster_info = [
      ["H2O cluster uptime: ", get_human_readable_time(cld["cloud_uptime_millis"])],
      ["H2O cluster version: ", cld["version"]],
      ["H2O cluster name: ", cld["cloud_name"]],
      ["H2O cluster total nodes: ", cld["cloud_size"]],
      ["H2O cluster total memory: ", get_human_readable_size(mmax)],
      ["H2O cluster total cores: ", str(ncpus)],
      ["H2O cluster allowed cores: ", str(allowed_cpus)],
      ["H2O cluster healthy: ", str(cluster_health)],
    ]
    print
    print tabulate.tabulate(cluster_info)
    print

    ver_h2o = cld['version']
    try:
      ver_pkg = pkg_resources.get_distribution("h2o").version
    except:
      ver_pkg = "UNKNOWN"

    if ver_h2o != ver_pkg:
      message = \
        "Version mismatch. H2O is version {0}, but the python package is version {1}.".format(ver_h2o, str(ver_pkg))
      if strict_version_check:
        raise EnvironmentError, message
      else:
        print "Warning: {0}".format(message)

  def _connect(self, size, max_retries=5, print_dots=False):
    """
    Does not actually "connect", instead simply tests that the cluster can be reached,
    is of a certain size, and is taking basic status commands.
    :param size: The number of H2O instances in the cloud.
    :return: The JSON response from a "stable" cluster.
    """
    max_retries = max_retries
    retries = 0
    while True:
      retries += 1
      if print_dots:
        self._print_dots(retries)
      try:
        cld = H2OConnection.get_json(url_suffix="Cloud")
        if not cld['cloud_healthy']:
          raise ValueError("Cluster reports unhealthy status", cld)
        if cld['cloud_size'] >= size and cld['consensus']:
          if print_dots: print " Connection sucessful!"
          return cld
      except EnvironmentError:
        pass
      # Cloud too small or voting in progress; sleep; try again
      time.sleep(0.1)
      if retries > max_retries:
        raise EnvironmentError("Max retries exceeded. Could not establish link to the H2O cloud @ " + str(self._ip) + ":" + str(self._port))

  def _print_dots(self, retries):
    sys.stdout.write("\rStarting H2O JVM and connecting: {}".format("." * retries))
    sys.stdout.flush()

  def _start_local_h2o_jar(self, mmax, mmin, ea, license, ice):
    command = H2OConnection._check_java()
    if license:
      if not os.path.exists(license):
        raise ValueError("License file not found (" + license + ")")

    if not ice:
      raise ValueError("`ice_root` must be specified")

    stdout = open(H2OConnection._tmp_file("stdout"), 'w')
    stderr = open(H2OConnection._tmp_file("stderr"), 'w')

    print "Using ice_root: " + ice
    print

    jar_file = os.path.join(sys.prefix, "h2o_jar/h2o.jar")

    jver = subprocess.check_output([command, "-version"], stderr=subprocess.STDOUT)

    print
    print "Java Version: " + jver
    print

    if "GNU libgcj" in jver:
      raise ValueError("Sorry, GNU Java is not supported for H2O.\n"+
                       "Please download the latest Java SE JDK 7 from the following URL:\n"+
                       "http://www.oracle.com/technetwork/java/javase/downloads/jdk7-downloads-1880260.html")

    if "Client VM" in jver:
      print "WARNING: "
      print "You have a 32-bit version of Java. H2O works best with 64-bit Java."
      print "Please download the latest Java SE JDK 7 from the following URL:"
      print "http://www.oracle.com/technetwork/java/javase/downloads/jdk7-downloads-1880260.html"
      print

    vm_opts = []
    if mmin: vm_opts += ["-Xms{}g".format(mmin)]
    if mmax: vm_opts += ["-Xmx{}g".format(mmax)]
    if ea:   vm_opts += ["-ea"]

    h2o_opts = ["-jar", jar_file,
                "-name", "H2O_started_from_python",
                "-ip", "127.0.0.1",
                "-port", "54321",
                "-ice_root", ice,
                ]
    if license:
      h2o_opts += ["-license", license]

    cmd = [command] + vm_opts + h2o_opts

    cwd = os.path.abspath(os.getcwd())
    self._child = subprocess.Popen(args=cmd, stdout=stdout, stderr=stderr, cwd=cwd)
    cld = self._connect(1, 30, True)
    return cld
{'fraction_non_alphanumeric': 0.10626865671641791, 'fraction_numerical': 0.015257048092868989, 'mean_word_length': 2.9321857068335944, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5290749', 'n_tokens_mistral': 5020, 'n_tokens_neox': 4831, 'n_words': 1482}