<commit_before># -*- encoding: utf-8 -*-
"""
h2o -- module for using H2O services.

:copyright: (c) 2016 H2O.ai
:license:   Apache License Version 2.0 (see LICENSE for details)
"""
from __future__ import absolute_import, division, print_function, unicode_literals

import logging
import os
import warnings
import webbrowser


from h2o.backend import H2OConnection
from h2o.backend import H2OConnectionConf
from h2o.backend import H2OLocalServer
from h2o.exceptions import H2OConnectionError, H2OValueError
from h2o.utils.config import H2OConfigReader
from h2o.utils.shared_utils import check_frame_id, deprecated, gen_header, py_tmp_key, quoted, urlopen
from h2o.utils.typechecks import assert_is_type, assert_satisfies, BoundInt, BoundNumeric, I, is_type, numeric, U
from .estimators.deeplearning import H2OAutoEncoderEstimator
from .estimators.deeplearning import H2ODeepLearningEstimator
from .estimators.deepwater import H2ODeepWaterEstimator
from .estimators.estimator_base import H2OEstimator
from .estimators.gbm import H2OGradientBoostingEstimator
from .estimators.glm import H2OGeneralizedLinearEstimator
from .estimators.glrm import H2OGeneralizedLowRankEstimator
from .estimators.kmeans import H2OKMeansEstimator
from .estimators.naive_bayes import H2ONaiveBayesEstimator
from .estimators.random_forest import H2ORandomForestEstimator
from .estimators.stackedensemble import H2OStackedEnsembleEstimator
from .expr import ExprNode
from .frame import H2OFrame
from .grid.grid_search import H2OGridSearch
from .job import H2OJob
from .model.model_base import ModelBase
from .transforms.decomposition import H2OPCA
from .transforms.decomposition import H2OSVD
from .utils.debugging import *  # NOQA
from .utils.compatibility import *  # NOQA
from .utils.compatibility import PY3

logging.basicConfig()

# An IPython deprecation warning is triggered after h2o.init(). Remove this once the deprecation has been resolved
warnings.filterwarnings('ignore', category=DeprecationWarning, module='.*/IPython/.*')


h2oconn = None  # type: H2OConnection

def connect(server=None, url=None, ip=None, port=None, https=None, verify_ssl_certificates=None, auth=None,
            proxy=None,cookies=None, verbose=True, config=None):
    """
    Connect to an existing H2O server, remote or local.

    There are two ways to connect to a server: either pass a `server` parameter containing an instance of
    an H2OLocalServer, or specify `ip` and `port` of the server that you want to connect to.

    :param server: An H2OLocalServer instance to connect to (optional).
    :param url: Full URL of the server to connect to (can be used instead of `ip` + `port` + `https`).
    :param ip: The ip address (or host name) of the server where H2O is running.
    :param port: Port number that H2O service is listening to.
    :param https: Set to True to connect via https:// instead of http://.
    :param verify_ssl_certificates: When using https, setting this to False will disable SSL certificates verification.
    :param auth: Either a (username, password) pair for basic authentication, or one of the requests.auth
                 authenticator objects.
    :param proxy: Proxy server address.
    :param cookies: Cookie (or list of) to add to request
    :param verbose: Set to False to disable printing connection status messages.
    :param connection_conf: Connection configuration object encapsulating connection parameters.
    :returns: the new :class:`H2OConnection` object.
    """
    global h2oconn
    if config:
        if "connect_params" in config:
            h2oconn = _connect_with_conf(config["connect_params"])
        else:
            h2oconn = _connect_with_conf(config)
    else:
        h2oconn = H2OConnection.open(server=server, url=url, ip=ip, port=port, https=https,
                                     auth=auth, verify_ssl_certificates=verify_ssl_certificates,
                                     proxy=proxy,cookies=cookies,
                                     verbose=verbose)
        h2oconn.cluster.timezone = "UTC"
        if verbose:
            h2oconn.cluster.show_status()
    return h2oconn


def api(endpoint, data=None, json=None, filename=None, save_to=None):
    """
    Perform a REST API request to a previously connected server.

    This function is mostly for internal purposes, but may occasionally be useful for direct access to
    the backend H2O server. It has same parameters as :meth:`H2OConnection.request <h2o.backend.H2OConnection.request>`.
    """
    # type checks are performed in H2OConnection class
    _check_connection()
    return h2oconn.request(endpoint, data=data, json=json, filename=filename, save_to=save_to)



def connection():
    """Return the current :class:`H2OConnection` handler."""
    return h2oconn


def version_check():
    """Used to verify that h2o-python module and the H2O server are compatible with each other."""
    from .__init__ import __version__ as ver_pkg
    ci = h2oconn.cluster
    if not ci:
        raise H2OConnectionError("Connection not initialized. Did you run h2o.connect()?")
    ver_h2o = ci.version
    if ver_pkg == "SUBST_PROJECT_VERSION": ver_pkg = "UNKNOWN"
    if str(ver_h2o) != str(ver_pkg):
        branch_name_h2o = ci.branch_name
        build_number_h2o = ci.build_number
        if build_number_h2o is None or build_number_h2o == "unknown":
            raise H2OConnectionError(
                "Version mismatch. H2O is version {0}, but the h2o-python package is version {1}. "
                "Upgrade H2O and h2o-Python to latest stable version - "
                "http://h2o-release.s3.amazonaws.com/h2o/latest_stable.html"
                "".format(ver_h2o, ver_pkg))
        elif build_number_h2o == "99999":
            raise H2OConnectionError(
                "Version mismatch. H2O is version {0}, but the h2o-python package is version {1}. "
                "This is a developer build, please contact your developer."
                "".format(ver_h2o, ver_pkg))
        else:
            raise H2OConnectionError(
                "Version mismatch. H2O is version {0}, but the h2o-python package is version {1}. "
                "Install the matching h2o-Python version from - "
                "http://h2o-release.s3.amazonaws.com/h2o/{2}/{3}/index.html."
                "".format(ver_h2o, ver_pkg, branch_name_h2o, build_number_h2o))
    # Check age of the install
    if ci.build_too_old:
        print("Warning: Your H2O cluster version is too old ({})! Please download and install the latest "
              "version from http://h2o.ai/download/".format(ci.build_age))


def init(url=None, ip=None, port=None, https=None, insecure=None, username=None, password=<PASSWORD>,
         cookies=None, proxy=None, start_h2o=True, nthreads=-1, ice_root=None, enable_assertions=True,
         max_mem_size=None, min_mem_size=None, strict_version_check=None, ignore_config=False, **kwargs):
    """
    Attempt to connect to a local server, or if not successful start a new server and connect to it.

    :param url: Full URL of the server to connect to (can be used instead of `ip` + `port` + `https`).
    :param ip: The ip address (or host name) of the server where H2O is running.
    :param port: Port number that H2O service is listening to.
    :param https: Set to True to connect via https:// instead of http://.
    :param insecure: When using https, setting this to True will disable SSL certificates verification.
    :param username: Username and
    :param password: Password for basic authentication.
    :param cookies: Cookie (or list of) to add to each request.
    :param proxy: Proxy server address.
    :param start_h2o: If False, do not attempt to start an h2o server when connection to an existing one failed.
    :param nthreads: "Number of threads" option when launching a new h2o server.
    :param ice_root: Directory for temporary files for the new h2o server.
    :param enable_assertions: Enable assertions in Java for the new h2o server.
    :param max_mem_size: Maximum memory to use for the new h2o server.
    :param min_mem_size: Minimum memory to use for the new h2o server.
    :param strict_version_check: If True, an error will be raised if the client and server versions don't match.
    :param ignore_config: Indicates whether a processing of a .h2oconfig file should be conducted or not. Default value is False.
    :param kwargs: (all other deprecated attributes)
    """
    global h2oconn
    assert_is_type(url, str, None)
    assert_is_type(ip, str, None)
    assert_is_type(port, int, str, None)
    assert_is_type(https, bool, None)
    assert_is_type(insecure, bool, None)
    assert_is_type(username, str, None)
    assert_is_type(password, str, None)
    assert_is_type(cookies, str, [str], None)
    assert_is_type(proxy, {str: str}, None)
    assert_is_type(start_h2o, bool, None)
    assert_is_type(nthreads, int)
    assert_is_type(ice_root, str, None)
    assert_is_type(enable_assertions, bool)
    assert_is_type(max_mem_size, int, str, None)
    assert_is_type(min_mem_size, int, str, None)
    assert_is_type(strict_version_check, bool, None)
    assert_is_type(kwargs, {"proxies": {str: str}, "max_mem_size_GB": int, "min_mem_size_GB": int,
                            "force_connect": bool})

    def get_mem_size(mmint, mmgb):
        if not mmint:  # treat 0 and "" as if they were None
            if mmgb is None: return None
            return mmgb << 30
        if is_type(mmint, int):
            # If the user gives some small number just assume it's in Gigabytes...
            if mmint < 1000: return mmint << 30
            return mmint
        if is_type(mmint, str):
            last = mmint[-1].upper()
            num = mmint[:-1]
            if not (num.isdigit() and last in "MGT"):
                raise H2OValueError("Wrong format for a *_memory_size argument: %s (should be a number followed by "
                                    "a suffix 'M', 'G' or 'T')" % mmint)
            if last == "T": return int(num) << 40
            if last == "G": return int(num) << 30
            if last == "M": return int(num) << 20

    scheme = "https" if https else "http"
    proxy = proxy[scheme] if proxy is not None and scheme in proxy else \
        kwargs["proxies"][scheme] if "proxies" in kwargs and scheme in kwargs["proxies"] else None
    mmax = get_mem_size(max_mem_size, kwargs.get("max_mem_size_GB"))
    mmin = get_mem_size(min_mem_size, kwargs.get("min_mem_size_GB"))
    auth = (username, password) if username and password else None
    check_version = True
    verify_ssl_certificates = True

    # Apply the config file if ignore_config=False
    if not ignore_config:
        config = H2OConfigReader.get_config()
        if url is None and ip is None and port is None and https is None and "init.url" in config:
            url = config["init.url"]
        if proxy is None and "init.proxy" in config:
            proxy = config["init.proxy"]
        if cookies is None and "init.cookies" in config:
            cookies = config["init.cookies"].split(";")
        if auth is None and "init.username" in config and "init.password" in config:
            auth = (config["init.username"], config["init.password"])
        if strict_version_check is None:
            if "init.check_version" in config:
                check_version = config["init.check_version"].lower() != "false"
            elif os.environ.get("H2O_DISABLE_STRICT_VERSION_CHECK"):
                check_version = False
        else:
            check_version = strict_version_check
        if insecure is None:
            if "init.verify_ssl_certificates" in config:
                verify_ssl_certificates = config["init.verify_ssl_certificates"].lower() != "false"
            else:
                verify_ssl_certificates = not insecure

    if not start_h2o:
        print("Warning: if you don't want to start local H2O server, then use of `h2o.connect()` is preferred.")
    if ip and ip != "localhost" and ip != "127.0.0.1" and start_h2o:
        print("Warning: connecting to remote server but falling back to local... Did you mean to use `h2o.connect()`?")
    try:
        h2oconn = H2OConnection.open(url=url, ip=ip, port=port, https=https,
                                     verify_ssl_certificates=verify_ssl_certificates,
                                     auth=auth, proxy=proxy,cookies=cookies, verbose=True,
                                     _msgs=("Checking whether there is an H2O instance running at {url}",
                                            "connected.", "not found."))
    except H2OConnectionError:
        # Backward compatibility: in init() port parameter really meant "baseport" when starting a local server...
        if port and not str(port).endswith("+"):
            port = str(port) + "+"
        if not start_h2o: raise
        hs = H2OLocalServer.start(nthreads=nthreads, enable_assertions=enable_assertions, max_mem_size=mmax,
                                  min_mem_size=mmin, ice_root=ice_root, port=port)
        h2oconn = H2OConnection.open(server=hs, https=https, verify_ssl_certificates=not insecure,
                                     auth=auth, proxy=proxy,cookies=cookies, verbose=True)
    if check_version:
        version_check()
    h2oconn.cluster.timezone = "UTC"
    h2oconn.cluster.show_status()


def lazy_import(path, pattern=None):
    """
    Import a single file or collection of files.

    :param path: A path to a data file (remote or local).
    :param pattern: Character string containing a regular expression to match file(s) in the folder.
    :returns: either a :class:`H2OFrame` with the content of the provided file, or a list of such frames if
        importing multiple files.
    """
    assert_is_type(path, str, [str])
    assert_is_type(pattern, str, None)
    if is_type(path, str):
        return _import(path, pattern)
    else:
        return [_import(p, pattern)[0] for p in path]


def _import(path, pattern):
    assert_is_type(path, str)
    assert_is_type(pattern, str, None)
    j = api("GET /3/ImportFiles", data={"path": path, "pattern": pattern})
    if j["fails"]: raise ValueError("ImportFiles of " + path + " failed on " + str(j["fails"]))
    return j["destination_frames"]


def upload_file(path, destination_frame=None, header=0, sep=None, col_names=None, col_types=None,
                na_strings=None):
    """
    Upload a dataset from the provided local path to the H2O cluster.

    Does a single-threaded push to H2O. Also see :meth:`import_file`.

    :param path: A path specifying the location of the data to upload.
    :param destination_frame:  The unique hex key assigned to the imported file. If none is given, a key will
        be automatically generated.
    :param header: -1 means the first line is data, 0 means guess, 1 means first line is header.
    :param sep: The field separator character. Values on each line of the file are separated by
        this character. If not provided, the parser will automatically detect the separator.
    :param col_names: A list of column names for the file.
    :param col_types: A list of types or a dictionary of column names to types to specify whether columns
        should be forced to a certain type upon import parsing. If a list, the types for elements that are
        one will be guessed. The possible types a column may have are:

        - "unknown" - this will force the column to be parsed as all NA
        - "uuid"    - the values in the column must be true UUID or will be parsed as NA
        - "string"  - force the column to be parsed as a string
        - "numeric" - force the column to be parsed as numeric. H2O will handle the compression of the numeric
          data in the optimal manner.
        - "enum"    - force the column to be parsed as a categorical column.
        - "time"    - force the column to be parsed as a time column. H2O will attempt to parse the following
          list of date time formats: (date) "yyyy-MM-dd", "yyyy MM dd", "dd-MMM-yy", "dd MMM yy", (time)
          "HH:mm:ss", "HH:mm:ss:SSS", "HH:mm:ss:SSSnnnnnn", "HH.mm.ss" "HH.mm.ss.SSS", "HH.mm.ss.SSSnnnnnn".
          Times can also contain "AM" or "PM".
    :param na_strings: A list of strings, or a list of lists of strings (one list per column), or a dictionary
        of column names to strings which are to be interpreted as missing values.

    :returns: a new :class:`H2OFrame` instance.

    :examples:
        >>> frame = h2o.upload_file("/path/to/local/data")
    """
    coltype = U(None, "unknown", "uuid", "string", "float", "real", "double", "int", "numeric",
                "categorical", "factor", "enum", "time")
    natype = U(str, [str])
    assert_is_type(path, str)
    assert_is_type(destination_frame, str, None)
    assert_is_type(header, -1, 0, 1)
    assert_is_type(sep, None, I(str, lambda s: len(s) == 1))
    assert_is_type(col_names, [str], None)
    assert_is_type(col_types, [coltype], {str: coltype}, None)
    assert_is_type(na_strings, [natype], {str: natype}, None)
    check_frame_id(destination_frame)
    if path.startswith("~"):
        path = os.path.expanduser(path)
    return H2OFrame()._upload_parse(path, destination_frame, header, sep, col_names, col_types, na_strings)


def import_file(path=None, destination_frame=None, parse=True, header=0, sep=None, col_names=None, col_types=None,
                na_strings=None, pattern=None):
    """
    Import a dataset that is already on the cluster.

    The path to the data must be a valid path for each node in the H2O cluster. If some node in the H2O cluster
    cannot see the file, then an exception will be thrown by the H2O cluster. Does a parallel/distributed
    multi-threaded pull of the data. The main difference between this method and :func:`upload_file` is that
    the latter works with local files, whereas this method imports remote files (i.e. files local to the server).
    If you running H2O server on your own maching, then both methods behave the same.

    :param path: path(s) specifying the location of the data to import or a path to a directory of files to import
    :param destination_frame: The unique hex key assigned to the imported file. If none is given, a key will be
        automatically generated.
    :param parse: If True, the file should be parsed after import.
    :param header: -1 means the first line is data, 0 means guess, 1 means first line is header.
    :param sep: The field separator character. Values on each line of the file are separated by
        this character. If not provided, the parser will automatically detect the separator.
    :param col_names: A list of column names for the file.
    :param col_types: A list of types or a dictionary of column names to types to specify whether columns
        should be forced to a certain type upon import parsing. If a list, the types for elements that are
        one will be guessed. The possible types a column may have are:

        - "unknown" - this will force the column to be parsed as all NA
        - "uuid"    - the values in the column must be true UUID or will be parsed as NA
        - "string"  - force the column to be parsed as a string
        - "numeric" - force the column to be parsed as numeric. H2O will handle the compression of the numeric
          data in the optimal manner.
        - "enum"    - force the column to be parsed as a categorical column.
        - "time"    - force the column to be parsed as a time column. H2O will attempt to parse the following
          list of date time formats: (date) "yyyy-MM-dd", "yyyy MM dd", "dd-MMM-yy", "dd MMM yy", (time)
          "HH:mm:ss", "HH:mm:ss:SSS", "HH:mm:ss:SSSnnnnnn", "HH.mm.ss" "HH.mm.ss.SSS", "HH.mm.ss.SSSnnnnnn".
          Times can also contain "AM" or "PM".
    :param na_strings: A list of strings, or a list of lists of strings (one list per column), or a dictionary
        of column names to strings which are to be interpreted as missing values.
    :param pattern: Character string containing a regular expression to match file(s) in the folder if `path` is a
        directory.

    :returns: a new :class:`H2OFrame` instance.

    :examples:
        >>> # Single file import
        >>> iris = import_file("h2o-3/smalldata/iris.csv")
        >>> # Return all files in the folder iris/ matching the regex r"iris_.*\.csv"
        >>> iris_pattern = h2o.import_file(path = "h2o-3/smalldata/iris",
        ...                                pattern = "iris_.*\.csv")
    """
    coltype = U(None, "unknown", "uuid", "string", "float", "real", "double", "int", "numeric",
                "categorical", "factor", "enum", "time")
    natype = U(str, [str])
    assert_is_type(path, str, [str])
    assert_is_type(pattern, str, None)
    assert_is_type(destination_frame, str, None)
    assert_is_type(parse, bool)
    assert_is_type(header, -1, 0, 1)
    assert_is_type(sep, None, I(str, lambda s: len(s) == 1))
    assert_is_type(col_names, [str], None)
    assert_is_type(col_types, [coltype], {str: coltype}, None)
    assert_is_type(na_strings, [natype], {str: natype}, None)
    check_frame_id(destination_frame)
    patharr = path if isinstance(path, list) else [path]
    if any(os.path.split(p)[0] == "~" for p in patharr):
        raise H2OValueError("Paths relative to a current user (~) are not valid in the server environment. "
                            "Please use absolute paths if possible.")
    if not parse:
        return lazy_import(path, pattern)
    else:
        return H2OFrame()._import_parse(path, pattern, destination_frame, header, sep, col_names, col_types, na_strings)


def import_sql_table(connection_url, table, username, password, columns=None, optimize=True):
    """
    Import SQL table to H2OFrame in memory.

    Assumes that the SQL table is not being updated and is stable.
    Runs multiple SELECT SQL queries concurrently for parallel ingestion.
    Be sure to start the h2o.jar in the terminal with your downloaded JDBC driver in the classpath::

        java -cp <path_to_h2o_jar>:<path_to_jdbc_driver_jar> water.H2OApp

    Also see :func:`import_sql_select`.
    Currently supported SQL databases are MySQL, PostgreSQL, and MariaDB. Support for Oracle 12g and Microsoft SQL
    Server is forthcoming.

    :param connection_url: URL of the SQL database connection as specified by the Java Database Connectivity (JDBC)
        Driver. For example, "jdbc:mysql://localhost:3306/menagerie?&useSSL=false"
    :param table: name of SQL table
    :param columns: a list of column names to import from SQL table. Default is to import all columns.
    :param username: username for SQL server
    :param password: password for SQL server
    :param optimize: optimize import of SQL table for faster imports. Experimental.

    :returns: an :class:`H2OFrame` containing data of the specified SQL table.

    :examples:
        >>> conn_url = "jdbc:mysql://172.16.2.178:3306/ingestSQL?&useSSL=false"
        >>> table = "citibike20k"
        >>> username = "root"
        >>> password = "<PASSWORD>"
        >>> my_citibike_data = h2o.import_sql_table(conn_url, table, username, password)
    """
    assert_is_type(connection_url, str)
    assert_is_type(table, str)
    assert_is_type(username, str)
    assert_is_type(password, str)
    assert_is_type(columns, [str], None)
    assert_is_type(optimize, bool)
    p = {"connection_url": connection_url, "table": table, "username": username, "password": password, "optimize": optimize}
    if columns:
        p["columns"] = ", ".join(columns)
    j = H2OJob(api("POST /99/ImportSQLTable", data=p), "Import SQL Table").poll()
    return get_frame(j.dest_key)


def import_sql_select(connection_url, select_query, username, password, optimize=True):
    """
    Import the SQL table that is the result of the specified SQL query to H2OFrame in memory.

    Creates a temporary SQL table from the specified sql_query.
    Runs multiple SELECT SQL queries on the temporary table concurrently for parallel ingestion, then drops the table.
    Be sure to start the h2o.jar in the terminal with your downloaded JDBC driver in the classpath::

      java -cp <path_to_h2o_jar>:<path_to_jdbc_driver_jar> water.H2OApp

    Also see h2o.import_sql_table. Currently supported SQL databases are MySQL, PostgreSQL, and MariaDB. Support
    for Oracle 12g and Microsoft SQL Server is forthcoming.

    :param connection_url: URL of the SQL database connection as specified by the Java Database Connectivity (JDBC)
        Driver. For example, "jdbc:mysql://localhost:3306/menagerie?&useSSL=false"
    :param select_query: SQL query starting with `SELECT` that returns rows from one or more database tables.
    :param username: username for SQL server
    :param password: password for SQL server
    :param optimize: optimize import of SQL table for faster imports. Experimental.

    :returns: an :class:`H2OFrame` containing data of the specified SQL query.

    :examples:
        >>> conn_url = "jdbc:mysql://172.16.2.178:3306/ingestSQL?&useSSL=false"
        >>> select_query = "SELECT bikeid from citibike20k"
        >>> username = "root"
        >>> password = "<PASSWORD>"
        >>> my_citibike_data = h2o.import_sql_select(conn_url, select_query,
        ...                                          username, password)
    """
    assert_is_type(connection_url, str)
    assert_is_type(select_query, str)
    assert_is_type(username, str)
    assert_is_type(password, str)
    assert_is_type(optimize, bool)
    p = {"connection_url": connection_url, "select_query": select_query, "username": username, "password": password,
         "optimize": optimize}
    j = H2OJob(api("POST /99/ImportSQLTable", data=p), "Import SQL Table").poll()
    return get_frame(j.dest_key)


def parse_setup(raw_frames, destination_frame=None, header=0, separator=None, column_names=None,
                column_types=None, na_strings=None):
    """
    Retrieve H2O's best guess as to what the structure of the data file is.

    During parse setup, the H2O cluster will make several guesses about the attributes of
    the data. This method allows a user to perform corrective measures by updating the
    returning dictionary from this method. This dictionary is then fed into `parse_raw` to
    produce the H2OFrame instance.

    :param raw_frames: a collection of imported file frames
    :param destination_frame: The unique hex key assigned to the imported file. If none is given, a key will
        automatically be generated.
    :param header: -1 means the first line is data, 0 means guess, 1 means first line is header.
    :param separator: The field separator character. Values on each line of the file are separated by
        this character. If not provided, the parser will automatically detect the separator.
    :param column_names: A list of column names for the file.
    :param column_types: A list of types or a dictionary of column names to types to specify whether columns
        should be forced to a certain type upon import parsing. If a list, the types for elements that are
        one will be guessed. The possible types a column may have are:

        - "unknown" - this will force the column to be parsed as all NA
        - "uuid"    - the values in the column must be true UUID or will be parsed as NA
        - "string"  - force the column to be parsed as a string
        - "numeric" - force the column to be parsed as numeric. H2O will handle the compression of the numeric
          data in the optimal manner.
        - "enum"    - force the column to be parsed as a categorical column.
        - "time"    - force the column to be parsed as a time column. H2O will attempt to parse the following
          list of date time formats: (date) "yyyy-MM-dd", "yyyy MM dd", "dd-MMM-yy", "dd MMM yy", (time)
          "HH:mm:ss", "HH:mm:ss:SSS", "HH:mm:ss:SSSnnnnnn", "HH.mm.ss" "HH.mm.ss.SSS", "HH.mm.ss.SSSnnnnnn".
          Times can also contain "AM" or "PM".

    :param na_strings: A list of strings, or a list of lists of strings (one list per column), or a dictionary
        of column names to strings which are to be interpreted as missing values.

    :returns: a dictionary containing parse parameters guessed by the H2O backend.
    """
    coltype = U(None, "unknown", "uuid", "string", "float", "real", "double", "int", "numeric",
                "categorical", "factor", "enum", "time")
    natype = U(str, [str])
    assert_is_type(raw_frames, str, [str])
    assert_is_type(destination_frame, None, str)
    assert_is_type(header, -1, 0, 1)
    assert_is_type(separator, None, I(str, lambda s: len(s) == 1))
    assert_is_type(column_names, [str], None)
    assert_is_type(column_types, [coltype], {str: coltype}, None)
    assert_is_type(na_strings, [natype], {str: natype}, None)
    check_frame_id(destination_frame)

    # The H2O backend only accepts things that are quoted
    if is_type(raw_frames, str): raw_frames = [raw_frames]

    # temporary dictionary just to pass the following information to the parser: header, separator
    kwargs = {"check_header": header, "source_frames": [quoted(frame_id) for frame_id in raw_frames]}
    if separator:
        kwargs["separator"] = ord(separator)

    j = api("POST /3/ParseSetup", data=kwargs)
    if "warnings" in j and j["warnings"]:
        for w in j["warnings"]:
            warnings.warn(w)
    # TODO: really should be url encoding...
    # TODO: clean up all this
    if destination_frame:
        j["destination_frame"] = destination_frame
    if column_names is not None:
        if not isinstance(column_names, list): raise ValueError("col_names should be a list")
        if len(column_names) != len(j["column_types"]): raise ValueError(
            "length of col_names should be equal to the number of columns: %d vs %d"
            % (len(column_names), len(j["column_types"])))
        j["column_names"] = column_names
    if column_types is not None:
        if isinstance(column_types, dict):
            # overwrite dictionary to ordered list of column types. if user didn't specify column type for all names,
            # use type provided by backend
            if j["column_names"] is None:  # no colnames discovered! (C1, C2, ...)
                j["column_names"] = gen_header(j["number_columns"])
            if not set(column_types.keys()).issubset(set(j["column_names"])): raise ValueError(
                "names specified in col_types is not a subset of the column names")
            idx = 0
            column_types_list = []
            for name in j["column_names"]:
                if name in column_types:
                    column_types_list.append(column_types[name])
                else:
                    column_types_list.append(j["column_types"][idx])
                idx += 1
            column_types = column_types_list
        elif isinstance(column_types, list):
            if len(column_types) != len(j["column_types"]): raise ValueError(
                "length of col_types should be equal to the number of columns")
            column_types = [column_types[i] if column_types[i] else j["column_types"][i] for i in
                            range(len(column_types))]
        else:  # not dictionary or list
            raise ValueError("col_types should be a list of types or a dictionary of column names to types")
        j["column_types"] = column_types
    if na_strings is not None:
        if isinstance(na_strings, dict):
            # overwrite dictionary to ordered list of lists of na_strings
            if not j["column_names"]: raise ValueError("column names should be specified")
            if not set(na_strings.keys()).issubset(set(j["column_names"])): raise ValueError(
                "names specified in na_strings is not a subset of the column names")
            j["na_strings"] = [[] for _ in range(len(j["column_names"]))]
            for name, na in na_strings.items():
                idx = j["column_names"].index(name)
                if is_type(na, str): na = [na]
                for n in na: j["na_strings"][idx].append(quoted(n))
        elif is_type(na_strings, [[str]]):
            if len(na_strings) != len(j["column_types"]):
                raise ValueError("length of na_strings should be equal to the number of columns")
            j["na_strings"] = [[quoted(na) for na in col] if col is not None else [] for col in na_strings]
        elif isinstance(na_strings, list):
            j["na_strings"] = [[quoted(na) for na in na_strings]] * len(j["column_types"])
        else:  # not a dictionary or list
            raise ValueError(
                "na_strings should be a list, a list of lists (one list per column), or a dictionary of column "
                "names to strings which are to be interpreted as missing values")

    # quote column names and column types also when not specified by user
    if j["column_names"]: j["column_names"] = list(map(quoted, j["column_names"]))
    j["column_types"] = list(map(quoted, j["column_types"]))
    return j


def parse_raw(setup, id=None, first_line_is_header=0):
    """
    Parse dataset using the parse setup structure.

    :param setup: Result of ``h2o.parse_setup()``
    :param id: an id for the frame.
    :param first_line_is_header: -1, 0, 1 if the first line is to be used as the header

    :returns: an :class:`H2OFrame` object.
    """
    assert_is_type(setup, dict)
    assert_is_type(id, str, None)
    assert_is_type(first_line_is_header, -1, 0, 1)
    check_frame_id(id)
    if id:
        setup["destination_frame"] = id
    if first_line_is_header != (-1, 0, 1):
        if first_line_is_header not in (-1, 0, 1): raise ValueError("first_line_is_header should be -1, 0, or 1")
        setup["check_header"] = first_line_is_header
    fr = H2OFrame()
    fr._parse_raw(setup)
    return fr


def assign(data, xid):
    """
    (internal) Assign new id to the frame.

    :param data: an H2OFrame whose id should be changed
    :param xid: new id for the frame.
    :returns: the passed frame.
    """
    assert_is_type(data, H2OFrame)
    assert_is_type(xid, str)
    assert_satisfies(xid, xid != data.frame_id)
    check_frame_id(xid)
    data._ex = ExprNode("assign", xid, data)._eval_driver(False)
    data._ex._cache._id = xid
    data._ex._children = None
    return data


def deep_copy(data, xid):
    """
    Create a deep clone of the frame ``data``.

    :param data: an H2OFrame to be cloned
    :param xid: (internal) id to be assigned to the new frame.
    :returns: new :class:`H2OFrame` which is the clone of the passed frame.
    """
    assert_is_type(data, H2OFrame)
    assert_is_type(xid, str)
    assert_satisfies(xid, xid != data.frame_id)
    check_frame_id(xid)
    duplicate = data.apply(lambda x: x)
    duplicate._ex = ExprNode("assign", xid, duplicate)._eval_driver(False)
    duplicate._ex._cache._id = xid
    duplicate._ex._children = None
    return duplicate


def get_model(model_id):
    """
    Load a model from the server.

    :param model_id: The model identification in H2O

    :returns: Model object, a subclass of H2OEstimator
    """
    assert_is_type(model_id, str)
    model_json = api("GET /3/Models/%s" % model_id)["models"][0]
    algo = model_json["algo"]
    if algo == "svd":            m = H2OSVD()
    elif algo == "pca":          m = H2OPCA()
    elif algo == "drf":          m = H2ORandomForestEstimator()
    elif algo == "naivebayes":   m = H2ONaiveBayesEstimator()
    elif algo == "kmeans":       m = H2OKMeansEstimator()
    elif algo == "glrm":         m = H2OGeneralizedLowRankEstimator()
    elif algo == "glm":          m = H2OGeneralizedLinearEstimator()
    elif algo == "gbm":          m = H2OGradientBoostingEstimator()
    elif algo == "deepwater":    m = H2ODeepWaterEstimator()
    elif algo == "deeplearning":
        if model_json["output"]["model_category"] == "AutoEncoder":
            m = H2OAutoEncoderEstimator()
        else:
            m = H2ODeepLearningEstimator()
    elif algo == "stackedensemble": m = H2OStackedEnsembleEstimator()
    else:
        raise ValueError("Unknown algo type: " + algo)
    m._resolve_model(model_id, model_json)
    return m


def get_grid(grid_id):
    """
    Return the specified grid.

    :param grid_id: The grid identification in h2o

    :returns: an :class:`H2OGridSearch` instance.
    """
    assert_is_type(grid_id, str)<commit_msg>Fix get_model to include XGBoost models.<commit_after># -*- encoding: utf-8 -*-
"""
h2o -- module for using H2O services.

:copyright: (c) 2016 H2O.ai
:license:   Apache License Version 2.0 (see LICENSE for details)
"""
from __future__ import absolute_import, division, print_function, unicode_literals

import logging
import os
import warnings
import webbrowser


from h2o.backend import H2OConnection
from h2o.backend import H2OConnectionConf
from h2o.backend import H2OLocalServer
from h2o.exceptions import H2OConnectionError, H2OValueError
from h2o.utils.config import H2OConfigReader
from h2o.utils.shared_utils import check_frame_id, deprecated, gen_header, py_tmp_key, quoted, urlopen
from h2o.utils.typechecks import assert_is_type, assert_satisfies, BoundInt, BoundNumeric, I, is_type, numeric, U
from .estimators.deeplearning import H2OAutoEncoderEstimator
from .estimators.deeplearning import H2ODeepLearningEstimator
from .estimators.deepwater import H2ODeepWaterEstimator
from .estimators.estimator_base import H2OEstimator
from .estimators import H2OXGBoostEstimator
from .estimators.gbm import H2OGradientBoostingEstimator
from .estimators.glm import H2OGeneralizedLinearEstimator
from .estimators.glrm import H2OGeneralizedLowRankEstimator
from .estimators.kmeans import H2OKMeansEstimator
from .estimators.naive_bayes import H2ONaiveBayesEstimator
from .estimators.random_forest import H2ORandomForestEstimator
from .estimators.stackedensemble import H2OStackedEnsembleEstimator
from .expr import ExprNode
from .frame import H2OFrame
from .grid.grid_search import H2OGridSearch
from .job import H2OJob
from .model.model_base import ModelBase
from .transforms.decomposition import H2OPCA
from .transforms.decomposition import H2OSVD
from .utils.debugging import *  # NOQA
from .utils.compatibility import *  # NOQA
from .utils.compatibility import PY3

logging.basicConfig()

# An IPython deprecation warning is triggered after h2o.init(). Remove this once the deprecation has been resolved
warnings.filterwarnings('ignore', category=DeprecationWarning, module='.*/IPython/.*')


h2oconn = None  # type: H2OConnection

def connect(server=None, url=None, ip=None, port=None, https=None, verify_ssl_certificates=None, auth=None,
            proxy=None,cookies=None, verbose=True, config=None):
    """
    Connect to an existing H2O server, remote or local.

    There are two ways to connect to a server: either pass a `server` parameter containing an instance of
    an H2OLocalServer, or specify `ip` and `port` of the server that you want to connect to.

    :param server: An H2OLocalServer instance to connect to (optional).
    :param url: Full URL of the server to connect to (can be used instead of `ip` + `port` + `https`).
    :param ip: The ip address (or host name) of the server where H2O is running.
    :param port: Port number that H2O service is listening to.
    :param https: Set to True to connect via https:// instead of http://.
    :param verify_ssl_certificates: When using https, setting this to False will disable SSL certificates verification.
    :param auth: Either a (username, password) pair for basic authentication, or one of the requests.auth
                 authenticator objects.
    :param proxy: Proxy server address.
    :param cookies: Cookie (or list of) to add to request
    :param verbose: Set to False to disable printing connection status messages.
    :param connection_conf: Connection configuration object encapsulating connection parameters.
    :returns: the new :class:`H2OConnection` object.
    """
    global h2oconn
    if config:
        if "connect_params" in config:
            h2oconn = _connect_with_conf(config["connect_params"])
        else:
            h2oconn = _connect_with_conf(config)
    else:
        h2oconn = H2OConnection.open(server=server, url=url, ip=ip, port=port, https=https,
                                     auth=auth, verify_ssl_certificates=verify_ssl_certificates,
                                     proxy=proxy,cookies=cookies,
                                     verbose=verbose)
        h2oconn.cluster.timezone = "UTC"
        if verbose:
            h2oconn.cluster.show_status()
    return h2oconn


def api(endpoint, data=None, json=None, filename=None, save_to=None):
    """
    Perform a REST API request to a previously connected server.

    This function is mostly for internal purposes, but may occasionally be useful for direct access to
    the backend H2O server. It has same parameters as :meth:`H2OConnection.request <h2o.backend.H2OConnection.request>`.
    """
    # type checks are performed in H2OConnection class
    _check_connection()
    return h2oconn.request(endpoint, data=data, json=json, filename=filename, save_to=save_to)



def connection():
    """Return the current :class:`H2OConnection` handler."""
    return h2oconn


def version_check():
    """Used to verify that h2o-python module and the H2O server are compatible with each other."""
    from .__init__ import __version__ as ver_pkg
    ci = h2oconn.cluster
    if not ci:
        raise H2OConnectionError("Connection not initialized. Did you run h2o.connect()?")
    ver_h2o = ci.version
    if ver_pkg == "SUBST_PROJECT_VERSION": ver_pkg = "UNKNOWN"
    if str(ver_h2o) != str(ver_pkg):
        branch_name_h2o = ci.branch_name
        build_number_h2o = ci.build_number
        if build_number_h2o is None or build_number_h2o == "unknown":
            raise H2OConnectionError(
                "Version mismatch. H2O is version {0}, but the h2o-python package is version {1}. "
                "Upgrade H2O and h2o-Python to latest stable version - "
                "http://h2o-release.s3.amazonaws.com/h2o/latest_stable.html"
                "".format(ver_h2o, ver_pkg))
        elif build_number_h2o == "99999":
            raise H2OConnectionError(
                "Version mismatch. H2O is version {0}, but the h2o-python package is version {1}. "
                "This is a developer build, please contact your developer."
                "".format(ver_h2o, ver_pkg))
        else:
            raise H2OConnectionError(
                "Version mismatch. H2O is version {0}, but the h2o-python package is version {1}. "
                "Install the matching h2o-Python version from - "
                "http://h2o-release.s3.amazonaws.com/h2o/{2}/{3}/index.html."
                "".format(ver_h2o, ver_pkg, branch_name_h2o, build_number_h2o))
    # Check age of the install
    if ci.build_too_old:
        print("Warning: Your H2O cluster version is too old ({})! Please download and install the latest "
              "version from http://h2o.ai/download/".format(ci.build_age))


def init(url=None, ip=None, port=None, https=None, insecure=None, username=None, password=<PASSWORD>,
         cookies=None, proxy=None, start_h2o=True, nthreads=-1, ice_root=None, enable_assertions=True,
         max_mem_size=None, min_mem_size=None, strict_version_check=None, ignore_config=False, **kwargs):
    """
    Attempt to connect to a local server, or if not successful start a new server and connect to it.

    :param url: Full URL of the server to connect to (can be used instead of `ip` + `port` + `https`).
    :param ip: The ip address (or host name) of the server where H2O is running.
    :param port: Port number that H2O service is listening to.
    :param https: Set to True to connect via https:// instead of http://.
    :param insecure: When using https, setting this to True will disable SSL certificates verification.
    :param username: Username and
    :param password: <PASSWORD>.
    :param cookies: Cookie (or list of) to add to each request.
    :param proxy: Proxy server address.
    :param start_h2o: If False, do not attempt to start an h2o server when connection to an existing one failed.
    :param nthreads: "Number of threads" option when launching a new h2o server.
    :param ice_root: Directory for temporary files for the new h2o server.
    :param enable_assertions: Enable assertions in Java for the new h2o server.
    :param max_mem_size: Maximum memory to use for the new h2o server.
    :param min_mem_size: Minimum memory to use for the new h2o server.
    :param strict_version_check: If True, an error will be raised if the client and server versions don't match.
    :param ignore_config: Indicates whether a processing of a .h2oconfig file should be conducted or not. Default value is False.
    :param kwargs: (all other deprecated attributes)
    """
    global h2oconn
    assert_is_type(url, str, None)
    assert_is_type(ip, str, None)
    assert_is_type(port, int, str, None)
    assert_is_type(https, bool, None)
    assert_is_type(insecure, bool, None)
    assert_is_type(username, str, None)
    assert_is_type(password, str, None)
    assert_is_type(cookies, str, [str], None)
    assert_is_type(proxy, {str: str}, None)
    assert_is_type(start_h2o, bool, None)
    assert_is_type(nthreads, int)
    assert_is_type(ice_root, str, None)
    assert_is_type(enable_assertions, bool)
    assert_is_type(max_mem_size, int, str, None)
    assert_is_type(min_mem_size, int, str, None)
    assert_is_type(strict_version_check, bool, None)
    assert_is_type(kwargs, {"proxies": {str: str}, "max_mem_size_GB": int, "min_mem_size_GB": int,
                            "force_connect": bool})

    def get_mem_size(mmint, mmgb):
        if not mmint:  # treat 0 and "" as if they were None
            if mmgb is None: return None
            return mmgb << 30
        if is_type(mmint, int):
            # If the user gives some small number just assume it's in Gigabytes...
            if mmint < 1000: return mmint << 30
            return mmint
        if is_type(mmint, str):
            last = mmint[-1].upper()
            num = mmint[:-1]
            if not (num.isdigit() and last in "MGT"):
                raise H2OValueError("Wrong format for a *_memory_size argument: %s (should be a number followed by "
                                    "a suffix 'M', 'G' or 'T')" % mmint)
            if last == "T": return int(num) << 40
            if last == "G": return int(num) << 30
            if last == "M": return int(num) << 20

    scheme = "https" if https else "http"
    proxy = proxy[scheme] if proxy is not None and scheme in proxy else \
        kwargs["proxies"][scheme] if "proxies" in kwargs and scheme in kwargs["proxies"] else None
    mmax = get_mem_size(max_mem_size, kwargs.get("max_mem_size_GB"))
    mmin = get_mem_size(min_mem_size, kwargs.get("min_mem_size_GB"))
    auth = (username, password) if username and password else None
    check_version = True
    verify_ssl_certificates = True

    # Apply the config file if ignore_config=False
    if not ignore_config:
        config = H2OConfigReader.get_config()
        if url is None and ip is None and port is None and https is None and "init.url" in config:
            url = config["init.url"]
        if proxy is None and "init.proxy" in config:
            proxy = config["init.proxy"]
        if cookies is None and "init.cookies" in config:
            cookies = config["init.cookies"].split(";")
        if auth is None and "init.username" in config and "init.password" in config:
            auth = (config["init.username"], config["init.password"])
        if strict_version_check is None:
            if "init.check_version" in config:
                check_version = config["init.check_version"].lower() != "false"
            elif os.environ.get("H2O_DISABLE_STRICT_VERSION_CHECK"):
                check_version = False
        else:
            check_version = strict_version_check
        if insecure is None:
            if "init.verify_ssl_certificates" in config:
                verify_ssl_certificates = config["init.verify_ssl_certificates"].lower() != "false"
            else:
                verify_ssl_certificates = not insecure

    if not start_h2o:
        print("Warning: if you don't want to start local H2O server, then use of `h2o.connect()` is preferred.")
    if ip and ip != "localhost" and ip != "127.0.0.1" and start_h2o:
        print("Warning: connecting to remote server but falling back to local... Did you mean to use `h2o.connect()`?")
    try:
        h2oconn = H2OConnection.open(url=url, ip=ip, port=port, https=https,
                                     verify_ssl_certificates=verify_ssl_certificates,
                                     auth=auth, proxy=proxy,cookies=cookies, verbose=True,
                                     _msgs=("Checking whether there is an H2O instance running at {url}",
                                            "connected.", "not found."))
    except H2OConnectionError:
        # Backward compatibility: in init() port parameter really meant "baseport" when starting a local server...
        if port and not str(port).endswith("+"):
            port = str(port) + "+"
        if not start_h2o: raise
        hs = H2OLocalServer.start(nthreads=nthreads, enable_assertions=enable_assertions, max_mem_size=mmax,
                                  min_mem_size=mmin, ice_root=ice_root, port=port)
        h2oconn = H2OConnection.open(server=hs, https=https, verify_ssl_certificates=not insecure,
                                     auth=auth, proxy=proxy,cookies=cookies, verbose=True)
    if check_version:
        version_check()
    h2oconn.cluster.timezone = "UTC"
    h2oconn.cluster.show_status()


def lazy_import(path, pattern=None):
    """
    Import a single file or collection of files.

    :param path: A path to a data file (remote or local).
    :param pattern: Character string containing a regular expression to match file(s) in the folder.
    :returns: either a :class:`H2OFrame` with the content of the provided file, or a list of such frames if
        importing multiple files.
    """
    assert_is_type(path, str, [str])
    assert_is_type(pattern, str, None)
    if is_type(path, str):
        return _import(path, pattern)
    else:
        return [_import(p, pattern)[0] for p in path]


def _import(path, pattern):
    assert_is_type(path, str)
    assert_is_type(pattern, str, None)
    j = api("GET /3/ImportFiles", data={"path": path, "pattern": pattern})
    if j["fails"]: raise ValueError("ImportFiles of " + path + " failed on " + str(j["fails"]))
    return j["destination_frames"]


def upload_file(path, destination_frame=None, header=0, sep=None, col_names=None, col_types=None,
                na_strings=None):
    """
    Upload a dataset from the provided local path to the H2O cluster.

    Does a single-threaded push to H2O. Also see :meth:`import_file`.

    :param path: A path specifying the location of the data to upload.
    :param destination_frame:  The unique hex key assigned to the imported file. If none is given, a key will
        be automatically generated.
    :param header: -1 means the first line is data, 0 means guess, 1 means first line is header.
    :param sep: The field separator character. Values on each line of the file are separated by
        this character. If not provided, the parser will automatically detect the separator.
    :param col_names: A list of column names for the file.
    :param col_types: A list of types or a dictionary of column names to types to specify whether columns
        should be forced to a certain type upon import parsing. If a list, the types for elements that are
        one will be guessed. The possible types a column may have are:

        - "unknown" - this will force the column to be parsed as all NA
        - "uuid"    - the values in the column must be true UUID or will be parsed as NA
        - "string"  - force the column to be parsed as a string
        - "numeric" - force the column to be parsed as numeric. H2O will handle the compression of the numeric
          data in the optimal manner.
        - "enum"    - force the column to be parsed as a categorical column.
        - "time"    - force the column to be parsed as a time column. H2O will attempt to parse the following
          list of date time formats: (date) "yyyy-MM-dd", "yyyy MM dd", "dd-MMM-yy", "dd MMM yy", (time)
          "HH:mm:ss", "HH:mm:ss:SSS", "HH:mm:ss:SSSnnnnnn", "HH.mm.ss" "HH.mm.ss.SSS", "HH.mm.ss.SSSnnnnnn".
          Times can also contain "AM" or "PM".
    :param na_strings: A list of strings, or a list of lists of strings (one list per column), or a dictionary
        of column names to strings which are to be interpreted as missing values.

    :returns: a new :class:`H2OFrame` instance.

    :examples:
        >>> frame = h2o.upload_file("/path/to/local/data")
    """
    coltype = U(None, "unknown", "uuid", "string", "float", "real", "double", "int", "numeric",
                "categorical", "factor", "enum", "time")
    natype = U(str, [str])
    assert_is_type(path, str)
    assert_is_type(destination_frame, str, None)
    assert_is_type(header, -1, 0, 1)
    assert_is_type(sep, None, I(str, lambda s: len(s) == 1))
    assert_is_type(col_names, [str], None)
    assert_is_type(col_types, [coltype], {str: coltype}, None)
    assert_is_type(na_strings, [natype], {str: natype}, None)
    check_frame_id(destination_frame)
    if path.startswith("~"):
        path = os.path.expanduser(path)
    return H2OFrame()._upload_parse(path, destination_frame, header, sep, col_names, col_types, na_strings)


def import_file(path=None, destination_frame=None, parse=True, header=0, sep=None, col_names=None, col_types=None,
                na_strings=None, pattern=None):
    """
    Import a dataset that is already on the cluster.

    The path to the data must be a valid path for each node in the H2O cluster. If some node in the H2O cluster
    cannot see the file, then an exception will be thrown by the H2O cluster. Does a parallel/distributed
    multi-threaded pull of the data. The main difference between this method and :func:`upload_file` is that
    the latter works with local files, whereas this method imports remote files (i.e. files local to the server).
    If you running H2O server on your own maching, then both methods behave the same.

    :param path: path(s) specifying the location of the data to import or a path to a directory of files to import
    :param destination_frame: The unique hex key assigned to the imported file. If none is given, a key will be
        automatically generated.
    :param parse: If True, the file should be parsed after import.
    :param header: -1 means the first line is data, 0 means guess, 1 means first line is header.
    :param sep: The field separator character. Values on each line of the file are separated by
        this character. If not provided, the parser will automatically detect the separator.
    :param col_names: A list of column names for the file.
    :param col_types: A list of types or a dictionary of column names to types to specify whether columns
        should be forced to a certain type upon import parsing. If a list, the types for elements that are
        one will be guessed. The possible types a column may have are:

        - "unknown" - this will force the column to be parsed as all NA
        - "uuid"    - the values in the column must be true UUID or will be parsed as NA
        - "string"  - force the column to be parsed as a string
        - "numeric" - force the column to be parsed as numeric. H2O will handle the compression of the numeric
          data in the optimal manner.
        - "enum"    - force the column to be parsed as a categorical column.
        - "time"    - force the column to be parsed as a time column. H2O will attempt to parse the following
          list of date time formats: (date) "yyyy-MM-dd", "yyyy MM dd", "dd-MMM-yy", "dd MMM yy", (time)
          "HH:mm:ss", "HH:mm:ss:SSS", "HH:mm:ss:SSSnnnnnn", "HH.mm.ss" "HH.mm.ss.SSS", "HH.mm.ss.SSSnnnnnn".
          Times can also contain "AM" or "PM".
    :param na_strings: A list of strings, or a list of lists of strings (one list per column), or a dictionary
        of column names to strings which are to be interpreted as missing values.
    :param pattern: Character string containing a regular expression to match file(s) in the folder if `path` is a
        directory.

    :returns: a new :class:`H2OFrame` instance.

    :examples:
        >>> # Single file import
        >>> iris = import_file("h2o-3/smalldata/iris.csv")
        >>> # Return all files in the folder iris/ matching the regex r"iris_.*\.csv"
        >>> iris_pattern = h2o.import_file(path = "h2o-3/smalldata/iris",
        ...                                pattern = "iris_.*\.csv")
    """
    coltype = U(None, "unknown", "uuid", "string", "float", "real", "double", "int", "numeric",
                "categorical", "factor", "enum", "time")
    natype = U(str, [str])
    assert_is_type(path, str, [str])
    assert_is_type(pattern, str, None)
    assert_is_type(destination_frame, str, None)
    assert_is_type(parse, bool)
    assert_is_type(header, -1, 0, 1)
    assert_is_type(sep, None, I(str, lambda s: len(s) == 1))
    assert_is_type(col_names, [str], None)
    assert_is_type(col_types, [coltype], {str: coltype}, None)
    assert_is_type(na_strings, [natype], {str: natype}, None)
    check_frame_id(destination_frame)
    patharr = path if isinstance(path, list) else [path]
    if any(os.path.split(p)[0] == "~" for p in patharr):
        raise H2OValueError("Paths relative to a current user (~) are not valid in the server environment. "
                            "Please use absolute paths if possible.")
    if not parse:
        return lazy_import(path, pattern)
    else:
        return H2OFrame()._import_parse(path, pattern, destination_frame, header, sep, col_names, col_types, na_strings)


def import_sql_table(connection_url, table, username, password, columns=None, optimize=True):
    """
    Import SQL table to H2OFrame in memory.

    Assumes that the SQL table is not being updated and is stable.
    Runs multiple SELECT SQL queries concurrently for parallel ingestion.
    Be sure to start the h2o.jar in the terminal with your downloaded JDBC driver in the classpath::

        java -cp <path_to_h2o_jar>:<path_to_jdbc_driver_jar> water.H2OApp

    Also see :func:`import_sql_select`.
    Currently supported SQL databases are MySQL, PostgreSQL, and MariaDB. Support for Oracle 12g and Microsoft SQL
    Server is forthcoming.

    :param connection_url: URL of the SQL database connection as specified by the Java Database Connectivity (JDBC)
        Driver. For example, "jdbc:mysql://localhost:3306/menagerie?&useSSL=false"
    :param table: name of SQL table
    :param columns: a list of column names to import from SQL table. Default is to import all columns.
    :param username: username for SQL server
    :param password: password for SQL server
    :param optimize: optimize import of SQL table for faster imports. Experimental.

    :returns: an :class:`H2OFrame` containing data of the specified SQL table.

    :examples:
        >>> conn_url = "jdbc:mysql://172.16.2.178:3306/ingestSQL?&useSSL=false"
        >>> table = "citibike20k"
        >>> username = "root"
        >>> password = "<PASSWORD>"
        >>> my_citibike_data = h2o.import_sql_table(conn_url, table, username, password)
    """
    assert_is_type(connection_url, str)
    assert_is_type(table, str)
    assert_is_type(username, str)
    assert_is_type(password, str)
    assert_is_type(columns, [str], None)
    assert_is_type(optimize, bool)
    p = {"connection_url": connection_url, "table": table, "username": username, "password": password, "optimize": optimize}
    if columns:
        p["columns"] = ", ".join(columns)
    j = H2OJob(api("POST /99/ImportSQLTable", data=p), "Import SQL Table").poll()
    return get_frame(j.dest_key)


def import_sql_select(connection_url, select_query, username, password, optimize=True):
    """
    Import the SQL table that is the result of the specified SQL query to H2OFrame in memory.

    Creates a temporary SQL table from the specified sql_query.
    Runs multiple SELECT SQL queries on the temporary table concurrently for parallel ingestion, then drops the table.
    Be sure to start the h2o.jar in the terminal with your downloaded JDBC driver in the classpath::

      java -cp <path_to_h2o_jar>:<path_to_jdbc_driver_jar> water.H2OApp

    Also see h2o.import_sql_table. Currently supported SQL databases are MySQL, PostgreSQL, and MariaDB. Support
    for Oracle 12g and Microsoft SQL Server is forthcoming.

    :param connection_url: URL of the SQL database connection as specified by the Java Database Connectivity (JDBC)
        Driver. For example, "jdbc:mysql://localhost:3306/menagerie?&useSSL=false"
    :param select_query: SQL query starting with `SELECT` that returns rows from one or more database tables.
    :param username: username for SQL server
    :param password: password for SQL server
    :param optimize: optimize import of SQL table for faster imports. Experimental.

    :returns: an :class:`H2OFrame` containing data of the specified SQL query.

    :examples:
        >>> conn_url = "jdbc:mysql://172.16.2.178:3306/ingestSQL?&useSSL=false"
        >>> select_query = "SELECT bikeid from citibike20k"
        >>> username = "root"
        >>> password = "<PASSWORD>"
        >>> my_citibike_data = h2o.import_sql_select(conn_url, select_query,
        ...                                          username, password)
    """
    assert_is_type(connection_url, str)
    assert_is_type(select_query, str)
    assert_is_type(username, str)
    assert_is_type(password, str)
    assert_is_type(optimize, bool)
    p = {"connection_url": connection_url, "select_query": select_query, "username": username, "password": password,
         "optimize": optimize}
    j = H2OJob(api("POST /99/ImportSQLTable", data=p), "Import SQL Table").poll()
    return get_frame(j.dest_key)


def parse_setup(raw_frames, destination_frame=None, header=0, separator=None, column_names=None,
                column_types=None, na_strings=None):
    """
    Retrieve H2O's best guess as to what the structure of the data file is.

    During parse setup, the H2O cluster will make several guesses about the attributes of
    the data. This method allows a user to perform corrective measures by updating the
    returning dictionary from this method. This dictionary is then fed into `parse_raw` to
    produce the H2OFrame instance.

    :param raw_frames: a collection of imported file frames
    :param destination_frame: The unique hex key assigned to the imported file. If none is given, a key will
        automatically be generated.
    :param header: -1 means the first line is data, 0 means guess, 1 means first line is header.
    :param separator: The field separator character. Values on each line of the file are separated by
        this character. If not provided, the parser will automatically detect the separator.
    :param column_names: A list of column names for the file.
    :param column_types: A list of types or a dictionary of column names to types to specify whether columns
        should be forced to a certain type upon import parsing. If a list, the types for elements that are
        one will be guessed. The possible types a column may have are:

        - "unknown" - this will force the column to be parsed as all NA
        - "uuid"    - the values in the column must be true UUID or will be parsed as NA
        - "string"  - force the column to be parsed as a string
        - "numeric" - force the column to be parsed as numeric. H2O will handle the compression of the numeric
          data in the optimal manner.
        - "enum"    - force the column to be parsed as a categorical column.
        - "time"    - force the column to be parsed as a time column. H2O will attempt to parse the following
          list of date time formats: (date) "yyyy-MM-dd", "yyyy MM dd", "dd-MMM-yy", "dd MMM yy", (time)
          "HH:mm:ss", "HH:mm:ss:SSS", "HH:mm:ss:SSSnnnnnn", "HH.mm.ss" "HH.mm.ss.SSS", "HH.mm.ss.SSSnnnnnn".
          Times can also contain "AM" or "PM".

    :param na_strings: A list of strings, or a list of lists of strings (one list per column), or a dictionary
        of column names to strings which are to be interpreted as missing values.

    :returns: a dictionary containing parse parameters guessed by the H2O backend.
    """
    coltype = U(None, "unknown", "uuid", "string", "float", "real", "double", "int", "numeric",
                "categorical", "factor", "enum", "time")
    natype = U(str, [str])
    assert_is_type(raw_frames, str, [str])
    assert_is_type(destination_frame, None, str)
    assert_is_type(header, -1, 0, 1)
    assert_is_type(separator, None, I(str, lambda s: len(s) == 1))
    assert_is_type(column_names, [str], None)
    assert_is_type(column_types, [coltype], {str: coltype}, None)
    assert_is_type(na_strings, [natype], {str: natype}, None)
    check_frame_id(destination_frame)

    # The H2O backend only accepts things that are quoted
    if is_type(raw_frames, str): raw_frames = [raw_frames]

    # temporary dictionary just to pass the following information to the parser: header, separator
    kwargs = {"check_header": header, "source_frames": [quoted(frame_id) for frame_id in raw_frames]}
    if separator:
        kwargs["separator"] = ord(separator)

    j = api("POST /3/ParseSetup", data=kwargs)
    if "warnings" in j and j["warnings"]:
        for w in j["warnings"]:
            warnings.warn(w)
    # TODO: really should be url encoding...
    # TODO: clean up all this
    if destination_frame:
        j["destination_frame"] = destination_frame
    if column_names is not None:
        if not isinstance(column_names, list): raise ValueError("col_names should be a list")
        if len(column_names) != len(j["column_types"]): raise ValueError(
            "length of col_names should be equal to the number of columns: %d vs %d"
            % (len(column_names), len(j["column_types"])))
        j["column_names"] = column_names
    if column_types is not None:
        if isinstance(column_types, dict):
            # overwrite dictionary to ordered list of column types. if user didn't specify column type for all names,
            # use type provided by backend
            if j["column_names"] is None:  # no colnames discovered! (C1, C2, ...)
                j["column_names"] = gen_header(j["number_columns"])
            if not set(column_types.keys()).issubset(set(j["column_names"])): raise ValueError(
                "names specified in col_types is not a subset of the column names")
            idx = 0
            column_types_list = []
            for name in j["column_names"]:
                if name in column_types:
                    column_types_list.append(column_types[name])
                else:
                    column_types_list.append(j["column_types"][idx])
                idx += 1
            column_types = column_types_list
        elif isinstance(column_types, list):
            if len(column_types) != len(j["column_types"]): raise ValueError(
                "length of col_types should be equal to the number of columns")
            column_types = [column_types[i] if column_types[i] else j["column_types"][i] for i in
                            range(len(column_types))]
        else:  # not dictionary or list
            raise ValueError("col_types should be a list of types or a dictionary of column names to types")
        j["column_types"] = column_types
    if na_strings is not None:
        if isinstance(na_strings, dict):
            # overwrite dictionary to ordered list of lists of na_strings
            if not j["column_names"]: raise ValueError("column names should be specified")
            if not set(na_strings.keys()).issubset(set(j["column_names"])): raise ValueError(
                "names specified in na_strings is not a subset of the column names")
            j["na_strings"] = [[] for _ in range(len(j["column_names"]))]
            for name, na in na_strings.items():
                idx = j["column_names"].index(name)
                if is_type(na, str): na = [na]
                for n in na: j["na_strings"][idx].append(quoted(n))
        elif is_type(na_strings, [[str]]):
            if len(na_strings) != len(j["column_types"]):
                raise ValueError("length of na_strings should be equal to the number of columns")
            j["na_strings"] = [[quoted(na) for na in col] if col is not None else [] for col in na_strings]
        elif isinstance(na_strings, list):
            j["na_strings"] = [[quoted(na) for na in na_strings]] * len(j["column_types"])
        else:  # not a dictionary or list
            raise ValueError(
                "na_strings should be a list, a list of lists (one list per column), or a dictionary of column "
                "names to strings which are to be interpreted as missing values")

    # quote column names and column types also when not specified by user
    if j["column_names"]: j["column_names"] = list(map(quoted, j["column_names"]))
    j["column_types"] = list(map(quoted, j["column_types"]))
    return j


def parse_raw(setup, id=None, first_line_is_header=0):
    """
    Parse dataset using the parse setup structure.

    :param setup: Result of ``h2o.parse_setup()``
    :param id: an id for the frame.
    :param first_line_is_header: -1, 0, 1 if the first line is to be used as the header

    :returns: an :class:`H2OFrame` object.
    """
    assert_is_type(setup, dict)
    assert_is_type(id, str, None)
    assert_is_type(first_line_is_header, -1, 0, 1)
    check_frame_id(id)
    if id:
        setup["destination_frame"] = id
    if first_line_is_header != (-1, 0, 1):
        if first_line_is_header not in (-1, 0, 1): raise ValueError("first_line_is_header should be -1, 0, or 1")
        setup["check_header"] = first_line_is_header
    fr = H2OFrame()
    fr._parse_raw(setup)
    return fr


def assign(data, xid):
    """
    (internal) Assign new id to the frame.

    :param data: an H2OFrame whose id should be changed
    :param xid: new id for the frame.
    :returns: the passed frame.
    """
    assert_is_type(data, H2OFrame)
    assert_is_type(xid, str)
    assert_satisfies(xid, xid != data.frame_id)
    check_frame_id(xid)
    data._ex = ExprNode("assign", xid, data)._eval_driver(False)
    data._ex._cache._id = xid
    data._ex._children = None
    return data


def deep_copy(data, xid):
    """
    Create a deep clone of the frame ``data``.

    :param data: an H2OFrame to be cloned
    :param xid: (internal) id to be assigned to the new frame.
    :returns: new :class:`H2OFrame` which is the clone of the passed frame.
    """
    assert_is_type(data, H2OFrame)
    assert_is_type(xid, str)
    assert_satisfies(xid, xid != data.frame_id)
    check_frame_id(xid)
    duplicate = data.apply(lambda x: x)
    duplicate._ex = ExprNode("assign", xid, duplicate)._eval_driver(False)
    duplicate._ex._cache._id = xid
    duplicate._ex._children = None
    return duplicate


def get_model(model_id):
    """
    Load a model from the server.

    :param model_id: The model identification in H2O

    :returns: Model object, a subclass of H2OEstimator
    """
    assert_is_type(model_id, str)
    model_json = api("GET /3/Models/%s" % model_id)["models"][0]
    algo = model_json["algo"]
    if algo == "svd":            m = H2OSVD()
    elif algo == "pca":          m = H2OPCA()
    elif algo == "drf":          m = H2ORandomForestEstimator()
    elif algo == "naivebayes":   m = H2ONaiveBayesEstimator()
    elif algo == "kmeans":       m = H2OKMeansEstimator()
    elif algo == "glrm":         m = H2OGeneralizedLowRankEstimator()
    elif algo == "glm":          m = H2OGeneralizedLinearEstimator()
    elif algo == "gbm":          m = H2OGradientBoostingEstimator()
    elif algo == "deepwater":    m = H2ODeepWaterEstimator()
    elif algo == "xgboost":      m = H2OXGBoostEstimator()
    elif algo == "deeplearning":
        if model_json["output"]["model_category"] == "AutoEncoder":
            m = H2OAutoEncoderEstimator()
        else:
            m = H2ODeepLearningEstimator()
    elif algo == "stackedensemble": m = H2OStackedEnsembleEstimator()
    else:
        raise ValueError("Unknown algo type: " + algo)
    m._resolve_model(model_id, model_json)
    return m


def get_grid(grid_id):
    """
    Return the specified grid.

    :param grid_id: The grid identification in h2o

    :returns: an :class:`H2OGridSearch` instance.
    """
    assert_is_type(grid_id, str)
{'fraction_non_alphanumeric': 0.09327332175055064, 'fraction_numerical': 0.00916590284142988, 'mean_word_length': 2.9902833124078825, 'pattern_counts': {'":': 73, '<': 42, '<?xml version=': 0, '>': 110, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4287124', 'n_tokens_mistral': 22124, 'n_tokens_neox': 21457, 'n_words': 7818}