<commit_before>import collections
from threading import Thread

import io
import jinja2
import logging
import os
import subprocess
import sys
import time
import yaml

from deploy.utils.constants import COMPONENT_PORTS, COMPONENTS_TO_OPEN_IN_BROWSER
from deploy.utils.kubectl_utils import get_pod_name, get_service_name, \
    run_in_pod, get_pod_status, get_node_name
from seqr.utils.network_utils import get_ip_address
from seqr.utils.shell_utils import run, wait_for, run_in_background

logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s')
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)


def get_component_port_pairs(components=[]):
    """Uses the PORTS dictionary to return a list of (<component name>, <port>) pairs (For example:
    [('postgres', 5432), ('seqr', 8000), ('seqr', 3000), ... ])

    Args:
        components (list): optional list of component names. If not specified, all components will be included.
    Returns:
        list of components
    """
    if not components:
        components = list(COMPONENT_PORTS.keys())

    return [(component, port) for component in components for port in COMPONENT_PORTS.get(component, [])]


def load_settings(settings_file_paths, settings=None, secrets=False):
    """Reads and parses the yaml settings file(s) and returns a dictionary of settings.
    These yaml files are treated as jinja templates. If a settings dictionary is also provided
    as an argument, it will be used as context for jinja template processing.

    Args:
        settings_file_paths (list): a list of yaml settings file paths to load
        settings (dict): optional dictionary of settings files
        secrets (bool): if False, the settings files are assumed to be yaml key-value pairs.
            if True, the files are parsed as Kubernetes Secrets files with base64-encoded values
    Return:
        dict: settings file containing all settings parsed from the given settings file
    """

    if settings is None:
        settings = collections.OrderedDict()

    for settings_path in settings_file_paths:
        with io.open(settings_path, encoding="UTF-8") as settings_file:
            try:
                settings_file_contents = settings_file.read()
                yaml_string = jinja2.Template(settings_file_contents).render(settings)
            except TypeError as e:
                raise ValueError('unable to render file: %(e)s' % locals())

            try:
                loaded_settings = yaml.load(yaml_string)
            except yaml.parser.ParserError as e:
                raise ValueError('unable to parse yaml file %(settings_path)s: %(e)s' % locals())

            if not loaded_settings:
                raise ValueError('yaml file %(settings_path)s appears to be empty' % locals())

            logger.info("Parsed %3d settings from %s" % (len(loaded_settings), settings_path))

            settings.update(loaded_settings)

    return settings


def render(input_base_dir, relative_file_path, settings, output_base_dir):
    """Calls the given render_func to convert the input file + settings dict to a rendered in-memory
    config which it then writes out to the output directory.

    Args:
        input_base_dir (string): The base directory for input file paths.
        relative_file_path (string): template file path relative to base_dir
        settings (dict): dictionary of key-value pairs for resolving any variables in the config template
        output_base_dir (string): The rendered config will be written to the file  {output_base_dir}/{relative_file_path}
    """

    input_file_path = os.path.join(input_base_dir, relative_file_path)
    with io.open(input_file_path, encoding="UTF-8") as istream:
        try:
            rendered_string = jinja2.Template(istream.read()).render(settings)
        except TypeError as e:
            raise ValueError('unable to render file: %(e)s' % locals())

    logger.info("Parsed %s" % relative_file_path)

    output_file_path = os.path.join(output_base_dir, relative_file_path)
    output_dir_path = os.path.dirname(output_file_path)

    if not os.path.isdir(output_dir_path):
        os.makedirs(output_dir_path)

    try:
        with open(output_file_path, 'w') as ostream:
            ostream.write(rendered_string)
    except Exception as e:
        logger.error("Couldn't write out %s" % relative_file_path)
        raise

    #os.chmod(output_file_path, 0x777)
    logger.info("-- wrote out %s" % output_file_path)


def retrieve_settings(deployment_target):
    settings = collections.OrderedDict()

    settings["HOME"] = os.path.expanduser("~")
    settings["TIMESTAMP"] = time.strftime("%Y%m%d_%H%M%S")
    settings["HOST_MACHINE_IP"] = get_ip_address()

    load_settings([
        "deploy/kubernetes/shared-settings.yaml",
        "deploy/kubernetes/%(deployment_target)s-settings.yaml" % locals(),
    ], settings)

    return settings


def check_kubernetes_context(deployment_target, set_if_different=False):
    """
    Make sure the environment is configured correctly, so that kubectl and other commands
    are actually aimed at the given deployment target and not some other cluster.

    Args:
        deployment_target (string): "local", "gcloud-dev", etc. See constants.DEPLOYMENT_TARGETS.
        set_if_different (bool): Update the context if the deployment_target doesn't match the current context.
    Return:
        string: The output of `kubectl config current-context`
    """
    try:
        cmd = 'kubectl config current-context'
        kubectl_current_context = subprocess.check_output(cmd, shell=True).strip()
    except subprocess.CalledProcessError as e:
        logger.error('Error while running "kubectl config current-context": %s', e)
        return

    context_is_different = False
    if deployment_target == "minikube":
        if (deployment_target == "minikube" and kubectl_current_context != "minikube"):
            logger.error((
                 "'%(cmd)s' returned '%(kubectl_current_context)s'. For %(deployment_target)s deployment, this is "
                 "expected to be '%(deployment_target)s'. Please configure your shell environment "
                 "to point to a local %(deployment_target)s cluster") % locals())
            context_is_different = True

    elif deployment_target.startswith("gcloud"):
        suffix = "-%s" % deployment_target.split("-")[-1]  # "dev" or "prod"
        if not kubectl_current_context.startswith('gke_') or suffix not in kubectl_current_context:
            logger.error(("'%(cmd)s' returned '%(kubectl_current_context)s' which doesn't match %(deployment_target)s. "
                          "To fix this, run:\n\n   "
                          "gcloud container clusters get-credentials <cluster-name>\n\n"
                          "Using one of these clusters: " + subprocess.check_output("gcloud container clusters list", shell=True) +
                          "\n\n") % locals())
            context_is_different = True
    else:
        raise ValueError("Unexpected value for deployment_target: %s" % deployment_target)

    if context_is_different:
        if set_if_different:
            set_environment(deployment_target)
        else:
            sys.exit(-1)

    return kubectl_current_context


def show_status():
    """Print status of various docker and kubernetes subsystems"""

    #run("docker info")
    #run("docker images")
    run("kubectl cluster-info", ignore_all_errors=True)
    #run("kubectl config view | grep 'username\|password'", ignore_all_errors=True)

    logger.info("==> Node IPs - for connecting to Kibana and elasticsearch via NodePorts 30002 and 30001:")
    run("kubectl describe nodes  | grep 'Name:\|ExternalIP'", ignore_all_errors=True)
    logger.info("==> elasticearch client IPs that hail can export to:")
    run("kubectl describe svc elasticsearch  | grep 'Name:\|Endpoints'", ignore_all_errors=True)

    run("kubectl get nodes", ignore_all_errors=True)
    run("kubectl get services --all-namespaces", ignore_all_errors=True)<commit_msg>added kubectl get deployments --all-namespaces to ./servctl status<commit_after>import collections
from threading import Thread

import io
import jinja2
import logging
import os
import subprocess
import sys
import time
import yaml

from deploy.utils.constants import COMPONENT_PORTS, COMPONENTS_TO_OPEN_IN_BROWSER
from deploy.utils.kubectl_utils import get_pod_name, get_service_name, \
    run_in_pod, get_pod_status, get_node_name
from seqr.utils.network_utils import get_ip_address
from seqr.utils.shell_utils import run, wait_for, run_in_background

logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s')
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)


def get_component_port_pairs(components=[]):
    """Uses the PORTS dictionary to return a list of (<component name>, <port>) pairs (For example:
    [('postgres', 5432), ('seqr', 8000), ('seqr', 3000), ... ])

    Args:
        components (list): optional list of component names. If not specified, all components will be included.
    Returns:
        list of components
    """
    if not components:
        components = list(COMPONENT_PORTS.keys())

    return [(component, port) for component in components for port in COMPONENT_PORTS.get(component, [])]


def load_settings(settings_file_paths, settings=None, secrets=False):
    """Reads and parses the yaml settings file(s) and returns a dictionary of settings.
    These yaml files are treated as jinja templates. If a settings dictionary is also provided
    as an argument, it will be used as context for jinja template processing.

    Args:
        settings_file_paths (list): a list of yaml settings file paths to load
        settings (dict): optional dictionary of settings files
        secrets (bool): if False, the settings files are assumed to be yaml key-value pairs.
            if True, the files are parsed as Kubernetes Secrets files with base64-encoded values
    Return:
        dict: settings file containing all settings parsed from the given settings file
    """

    if settings is None:
        settings = collections.OrderedDict()

    for settings_path in settings_file_paths:
        with io.open(settings_path, encoding="UTF-8") as settings_file:
            try:
                settings_file_contents = settings_file.read()
                yaml_string = jinja2.Template(settings_file_contents).render(settings)
            except TypeError as e:
                raise ValueError('unable to render file: %(e)s' % locals())

            try:
                loaded_settings = yaml.load(yaml_string)
            except yaml.parser.ParserError as e:
                raise ValueError('unable to parse yaml file %(settings_path)s: %(e)s' % locals())

            if not loaded_settings:
                raise ValueError('yaml file %(settings_path)s appears to be empty' % locals())

            logger.info("Parsed %3d settings from %s" % (len(loaded_settings), settings_path))

            settings.update(loaded_settings)

    return settings


def render(input_base_dir, relative_file_path, settings, output_base_dir):
    """Calls the given render_func to convert the input file + settings dict to a rendered in-memory
    config which it then writes out to the output directory.

    Args:
        input_base_dir (string): The base directory for input file paths.
        relative_file_path (string): template file path relative to base_dir
        settings (dict): dictionary of key-value pairs for resolving any variables in the config template
        output_base_dir (string): The rendered config will be written to the file  {output_base_dir}/{relative_file_path}
    """

    input_file_path = os.path.join(input_base_dir, relative_file_path)
    with io.open(input_file_path, encoding="UTF-8") as istream:
        try:
            rendered_string = jinja2.Template(istream.read()).render(settings)
        except TypeError as e:
            raise ValueError('unable to render file: %(e)s' % locals())

    logger.info("Parsed %s" % relative_file_path)

    output_file_path = os.path.join(output_base_dir, relative_file_path)
    output_dir_path = os.path.dirname(output_file_path)

    if not os.path.isdir(output_dir_path):
        os.makedirs(output_dir_path)

    try:
        with open(output_file_path, 'w') as ostream:
            ostream.write(rendered_string)
    except Exception as e:
        logger.error("Couldn't write out %s" % relative_file_path)
        raise

    #os.chmod(output_file_path, 0x777)
    logger.info("-- wrote out %s" % output_file_path)


def retrieve_settings(deployment_target):
    settings = collections.OrderedDict()

    settings["HOME"] = os.path.expanduser("~")
    settings["TIMESTAMP"] = time.strftime("%Y%m%d_%H%M%S")
    settings["HOST_MACHINE_IP"] = get_ip_address()

    load_settings([
        "deploy/kubernetes/shared-settings.yaml",
        "deploy/kubernetes/%(deployment_target)s-settings.yaml" % locals(),
    ], settings)

    return settings


def check_kubernetes_context(deployment_target, set_if_different=False):
    """
    Make sure the environment is configured correctly, so that kubectl and other commands
    are actually aimed at the given deployment target and not some other cluster.

    Args:
        deployment_target (string): "local", "gcloud-dev", etc. See constants.DEPLOYMENT_TARGETS.
        set_if_different (bool): Update the context if the deployment_target doesn't match the current context.
    Return:
        string: The output of `kubectl config current-context`
    """
    try:
        cmd = 'kubectl config current-context'
        kubectl_current_context = subprocess.check_output(cmd, shell=True).strip()
    except subprocess.CalledProcessError as e:
        logger.error('Error while running "kubectl config current-context": %s', e)
        return

    context_is_different = False
    if deployment_target == "minikube":
        if (deployment_target == "minikube" and kubectl_current_context != "minikube"):
            logger.error((
                 "'%(cmd)s' returned '%(kubectl_current_context)s'. For %(deployment_target)s deployment, this is "
                 "expected to be '%(deployment_target)s'. Please configure your shell environment "
                 "to point to a local %(deployment_target)s cluster") % locals())
            context_is_different = True

    elif deployment_target.startswith("gcloud"):
        suffix = "-%s" % deployment_target.split("-")[-1]  # "dev" or "prod"
        if not kubectl_current_context.startswith('gke_') or suffix not in kubectl_current_context:
            logger.error(("'%(cmd)s' returned '%(kubectl_current_context)s' which doesn't match %(deployment_target)s. "
                          "To fix this, run:\n\n   "
                          "gcloud container clusters get-credentials <cluster-name>\n\n"
                          "Using one of these clusters: " + subprocess.check_output("gcloud container clusters list", shell=True) +
                          "\n\n") % locals())
            context_is_different = True
    else:
        raise ValueError("Unexpected value for deployment_target: %s" % deployment_target)

    if context_is_different:
        if set_if_different:
            set_environment(deployment_target)
        else:
            sys.exit(-1)

    return kubectl_current_context


def show_status():
    """Print status of various docker and kubernetes subsystems"""

    #run("docker info")
    #run("docker images")
    run("kubectl cluster-info", ignore_all_errors=True)
    #run("kubectl config view | grep 'username\|password'", ignore_all_errors=True)

    logger.info("==> Node IPs - for connecting to Kibana and elasticsearch via NodePorts 30002 and 30001:")
    run("kubectl describe nodes  | grep 'Name:\|ExternalIP'", ignore_all_errors=True)
    logger.info("==> elasticearch client IPs that hail can export to:")
    run("kubectl describe svc elasticsearch  | grep 'Name:\|Endpoints'", ignore_all_errors=True)

    run("kubectl get nodes", ignore_all_errors=True)
    run("kubectl get deployments --all-namespaces", ignore_all_errors=True)
    run("kubectl get services --all-namespaces", ignore_all_errors=True)
{'fraction_non_alphanumeric': 0.09411330503782986, 'fraction_numerical': 0.00455188534169896, 'mean_word_length': 3.277295448566167, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2108439', 'n_tokens_mistral': 4797, 'n_tokens_neox': 4595, 'n_words': 1373}