<commit_before>#!/usr/bin/env python

import copy
import logging
import os
import socket
import sys

import yaml


DEFAULT_SOA_DIR = "/nail/etc/services"
DEFAULT_EXTRA_SOA_DIR = "/nail/etc/extra_soa/"
log = logging.getLogger(__name__)
_yaml_cache = {}
_use_yaml_cache = True

def enable_yaml_cache():
    global _use_yaml_cache
    _use_yaml_cache = True

def disable_yaml_cache():
    global _use_yaml_cache
    _use_yaml_cache = False

def read_port(port_file):
    # Try to read port information
    try:
        with open(port_file, 'r') as port_file_fd:
            port = int(port_file_fd.read().strip())
    except IOError:
        port = None
    except ValueError:
        port = None
    return port

def read_vip(vip_file):
    try:
        with open(vip_file, 'r') as vip_file_fd:
            vip = vip_file_fd.read().strip()
    except IOError:
        vip = None
    return vip

def load_yaml(fd):
    if yaml.__with_libyaml__:
        return yaml.load(fd, Loader=yaml.CLoader)
    else:
        return yaml.load(fd)

def read_lb_extras(lb_extras_file):
    return _read_yaml_file(lb_extras_file)

def read_monitoring(monitoring_file):
    return _read_yaml_file(monitoring_file)

def read_deploy(deploy_file):
    return _read_yaml_file(deploy_file)

def read_data(data_file):
    return _read_yaml_file(data_file)

def read_smartstack(smartstack_file):
    return _read_yaml_file(smartstack_file)

def read_service_information(service_file):
    return _read_yaml_file(service_file)

def _read_yaml_file(file_name):
    if _use_yaml_cache and file_name in _yaml_cache:
        return copy.deepcopy(_yaml_cache[file_name])
    data = {}
    try:
        with open(file_name, 'r') as fd:
            data = load_yaml(fd.read())
            data = data or {}
            if _use_yaml_cache:
                _yaml_cache[file_name] = data
    except IOError:
        pass
    except:
        print >>sys.stderr, "Failed to parse YAML from %s" % file_name
        raise
    return data

def generate_service_info(service_information, **kwargs):
    service_info = kwargs
    service_info.update(service_information)
    return service_info

def read_extra_service_information(service_name, extra_info, soa_dir=DEFAULT_SOA_DIR):
    return _read_yaml_file(os.path.join(
        os.path.abspath(soa_dir), service_name, extra_info + ".yaml"))

def read_service_configuration_from_dir(rootdir, service_dirname):
    port_file = os.path.join(rootdir, service_dirname, "port")
    vip_file = os.path.join(rootdir, service_dirname, "vip")
    lb_extras_file = os.path.join(rootdir, service_dirname, "lb.yaml")
    monitoring_file = os.path.join(rootdir, service_dirname, "monitoring.yaml")
    deploy_file = os.path.join(rootdir, service_dirname, "deploy.yaml")
    data_file = os.path.join(rootdir, service_dirname, "data.yaml")
    smartstack_file = os.path.join(rootdir, service_dirname, "smartstack.yaml")
    service_file = os.path.join(rootdir, service_dirname, "service.yaml")

    port = read_port(port_file)
    vip = read_vip(vip_file)
    lb_extras = read_lb_extras(lb_extras_file)
    monitoring = read_monitoring(monitoring_file)
    deploy = read_deploy(deploy_file)
    data = read_data(data_file)
    smartstack = read_smartstack(smartstack_file)
    service_information = read_service_information(service_file)

    return generate_service_info(
        service_information,
        port=port,
        vip=vip,
        lb_extras=lb_extras,
        monitoring=monitoring,
        deploy=deploy,
        data=data,
        smartstack=smartstack,
    )

def read_service_configuration(service_name, soa_dir=DEFAULT_SOA_DIR):
    return read_service_configuration_from_dir(os.path.abspath(soa_dir), service_name)

def read_services_configuration(soa_dir=DEFAULT_SOA_DIR):
    # Returns a dict of service information, keys are the service name
    # Not all services have all fields. Who knows what might be in there
    # You can't depend on every service having a vip, for example
    all_services = {}
    rootdir = os.path.abspath(soa_dir)
    for service_dirname in os.listdir(rootdir):
        service_name = service_dirname
        service_info = read_service_configuration_from_dir(rootdir, service_dirname)
        all_services.update( { service_name: service_info } )
    return all_services

def get_service_from_port(port, all_services=None):
    """
    Gets the name of the service from the port
    all_services allows you to feed in the services to look through
    
    Returns the name of the service
    """
    if port is None and not isinstance(port, int):
        return None

    if all_services is None:
        all_services = read_services_configuration()

    for name, info in all_services.items():
        if info.has_key('port') and port == info['port']:
            return name

def _list_extra_soa(action, extra_soa_dir):
    # This list includes additional services that we want to run on
    # the instance.
    rootdir = os.path.join(os.path.abspath(extra_soa_dir), action)
    # We expect to have a file for each service, but some instances may not
    # have any extra service and also missing the base directory.
    # In that case we return an empty list.
    if os.path.isdir(rootdir):
        return os.listdir(rootdir)
    else:
        return []

def list_extra_run_here(extra_soa_dir=DEFAULT_EXTRA_SOA_DIR):
    return _list_extra_soa('run_here', extra_soa_dir)

def services_that_run_here():
    hostname = socket.getfqdn()<commit_msg>fixed bad input if statement and made doc string more desciptive<commit_after>#!/usr/bin/env python

import copy
import logging
import os
import socket
import sys

import yaml


DEFAULT_SOA_DIR = "/nail/etc/services"
DEFAULT_EXTRA_SOA_DIR = "/nail/etc/extra_soa/"
log = logging.getLogger(__name__)
_yaml_cache = {}
_use_yaml_cache = True

def enable_yaml_cache():
    global _use_yaml_cache
    _use_yaml_cache = True

def disable_yaml_cache():
    global _use_yaml_cache
    _use_yaml_cache = False

def read_port(port_file):
    # Try to read port information
    try:
        with open(port_file, 'r') as port_file_fd:
            port = int(port_file_fd.read().strip())
    except IOError:
        port = None
    except ValueError:
        port = None
    return port

def read_vip(vip_file):
    try:
        with open(vip_file, 'r') as vip_file_fd:
            vip = vip_file_fd.read().strip()
    except IOError:
        vip = None
    return vip

def load_yaml(fd):
    if yaml.__with_libyaml__:
        return yaml.load(fd, Loader=yaml.CLoader)
    else:
        return yaml.load(fd)

def read_lb_extras(lb_extras_file):
    return _read_yaml_file(lb_extras_file)

def read_monitoring(monitoring_file):
    return _read_yaml_file(monitoring_file)

def read_deploy(deploy_file):
    return _read_yaml_file(deploy_file)

def read_data(data_file):
    return _read_yaml_file(data_file)

def read_smartstack(smartstack_file):
    return _read_yaml_file(smartstack_file)

def read_service_information(service_file):
    return _read_yaml_file(service_file)

def _read_yaml_file(file_name):
    if _use_yaml_cache and file_name in _yaml_cache:
        return copy.deepcopy(_yaml_cache[file_name])
    data = {}
    try:
        with open(file_name, 'r') as fd:
            data = load_yaml(fd.read())
            data = data or {}
            if _use_yaml_cache:
                _yaml_cache[file_name] = data
    except IOError:
        pass
    except:
        print >>sys.stderr, "Failed to parse YAML from %s" % file_name
        raise
    return data

def generate_service_info(service_information, **kwargs):
    service_info = kwargs
    service_info.update(service_information)
    return service_info

def read_extra_service_information(service_name, extra_info, soa_dir=DEFAULT_SOA_DIR):
    return _read_yaml_file(os.path.join(
        os.path.abspath(soa_dir), service_name, extra_info + ".yaml"))

def read_service_configuration_from_dir(rootdir, service_dirname):
    port_file = os.path.join(rootdir, service_dirname, "port")
    vip_file = os.path.join(rootdir, service_dirname, "vip")
    lb_extras_file = os.path.join(rootdir, service_dirname, "lb.yaml")
    monitoring_file = os.path.join(rootdir, service_dirname, "monitoring.yaml")
    deploy_file = os.path.join(rootdir, service_dirname, "deploy.yaml")
    data_file = os.path.join(rootdir, service_dirname, "data.yaml")
    smartstack_file = os.path.join(rootdir, service_dirname, "smartstack.yaml")
    service_file = os.path.join(rootdir, service_dirname, "service.yaml")

    port = read_port(port_file)
    vip = read_vip(vip_file)
    lb_extras = read_lb_extras(lb_extras_file)
    monitoring = read_monitoring(monitoring_file)
    deploy = read_deploy(deploy_file)
    data = read_data(data_file)
    smartstack = read_smartstack(smartstack_file)
    service_information = read_service_information(service_file)

    return generate_service_info(
        service_information,
        port=port,
        vip=vip,
        lb_extras=lb_extras,
        monitoring=monitoring,
        deploy=deploy,
        data=data,
        smartstack=smartstack,
    )

def read_service_configuration(service_name, soa_dir=DEFAULT_SOA_DIR):
    return read_service_configuration_from_dir(os.path.abspath(soa_dir), service_name)

def read_services_configuration(soa_dir=DEFAULT_SOA_DIR):
    # Returns a dict of service information, keys are the service name
    # Not all services have all fields. Who knows what might be in there
    # You can't depend on every service having a vip, for example
    all_services = {}
    rootdir = os.path.abspath(soa_dir)
    for service_dirname in os.listdir(rootdir):
        service_name = service_dirname
        service_info = read_service_configuration_from_dir(rootdir, service_dirname)
        all_services.update( { service_name: service_info } )
    return all_services

def get_service_from_port(port, all_services=None):
    """Gets the name of the service from the port
    all_services allows you to feed in the services to look through, pass
    in a dict of service names to service information eg.
    {
        'service_name': {
            'port': port_number
        }
    }
    
    Returns the name of the service
    """
    if port is None or not isinstance(port, int):
        return None

    if all_services is None:
        all_services = read_services_configuration()

    for name, info in all_services.items():
        if info.has_key('port') and port == info['port']:
            return name

def _list_extra_soa(action, extra_soa_dir):
    # This list includes additional services that we want to run on
    # the instance.
    rootdir = os.path.join(os.path.abspath(extra_soa_dir), action)
    # We expect to have a file for each service, but some instances may not
    # have any extra service and also missing the base directory.
    # In that case we return an empty list.
    if os.path.isdir(rootdir):
        return os.listdir(rootdir)
    else:
        return []

def list_extra_run_here(extra_soa_dir=DEFAULT_EXTRA_SOA_DIR):
    return _list_extra_soa('run_here', extra_soa_dir)

def services_that_run_here():
    hostname = socket.getfqdn()
{'fraction_non_alphanumeric': 0.08231189048939787, 'fraction_numerical': 0.0, 'mean_word_length': 3.5494505494505493, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '233838', 'n_tokens_mistral': 3645, 'n_tokens_neox': 3614, 'n_words': 851}