<commit_before>'''
Custom dynamic inventory script for Ansible, in Python.

This script will read in a configuration file either locally or fetched via HTTP and will
output a JSON data structure describing the inventory by merging the files as listed in
the config file.

Files can be in either YAML or JSON format
'''

# pylint: disable=wrong-import-order
# Support python 2 and 3
from __future__ import (absolute_import)
from future import standard_library
standard_library.install_aliases()
from builtins import object # pylint: disable=redefined-builtin

# stdlib
import json
import os
import sys

## This is the python3 name, formerly urlparse, install_aliases() above make
## this work under python2
from urllib.parse import (urlparse, urljoin) # pylint: disable=import-error

# extras from packages
import click
import dpath.util
import requests
import yaml

# Ansible stuff for Secrets
import ansible.parsing.vault as vault
from ansible.module_utils._text import to_text


def get_environment():
    '''Determine the platform/environment name from name of called script'''
    myname = os.path.basename(sys.argv[0])
    if myname != 'inventory':
        return myname
    else:
        return None


def create_links(environments, directory):
    '''Create symlinks for platform inventories'''
    for ename in environments:
        try:
            os.symlink(
                os.path.relpath(__file__, directory),
                os.path.join(directory, ename)
            )
        except(OSError) as err:
            click.echo("This symlink might already exist. Leaving it unchanged. Error: %s" % (err))


def fetch_data_remote(url, requestsobj=requests):
    '''fetch data from url and return as plaintext'''
    response = requestsobj.get(url)
    if response.status_code == 404:
        raise LookupError("Failed to find data at: {}".format(url))
    response.raise_for_status()
    return response.text


def fetch_data_local(localfile):
    '''Fetch data from local file and return as plaintext'''
    this_path = os.path.realpath(os.path.expanduser(localfile))
    if not os.path.exists(this_path):
        raise IOError("The file %s was not found" % this_path)
    try:
        f_obj = open(this_path, "rb")
        data = f_obj.read().strip()
        f_obj.close()
    except (OSError, IOError) as err:
        raise LookupError("Could not read file %s: %s" % (this_path, err))
    return to_text(data, errors='surrogate_or_strict')


def fetch_data(uri, requestsobj=requests, data_type=None, vault_password=None):
    '''Fetch data using either local or remote functions'''
    uriobj = urlparse(uri)
    loader = {
        'json': getattr(json, 'loads'),
        'yaml': getattr(yaml, 'load')
    }
    if data_type is None:
        # guess the data type from file extension
        if uriobj.path.endswith(('.yaml', '.yml')):
            data_type = 'yaml'
        else:
            data_type = os.path.splitext(uriobj.path)[1]
    data_type = data_type.lower()
    if data_type not in loader:
        raise AttributeError("Unsupported data type: {}".format(data_type))
    parser = loader[data_type]
    if uriobj.scheme in ['file', '']:
        data = fetch_data_local(uriobj.path)
    elif uriobj.scheme in ['http', 'https']:
        data = fetch_data_remote(uri, requestsobj)
    else:
        raise AttributeError("unsupported URI '{}'".format(uri))
    if vault.is_encrypted(data):
        if vault_password is None:
            data = '{}'
        else:
            data = vault.VaultLib(vault_password).decrypt(data)
    return parser(data)


def raise_for_type(item, types, section):
    '''raise an AttributeError if `item` is not of `types`'''
    type_names = None
    if isinstance(types, tuple):
        type_names = [t.__name__ for t in types]
    elif isinstance(types, type):
        type_names = types.__name__
    else:
        raise AttributeError(
            "Invalid type '{}' for `types` parameter, must be type or tuple of types".format(
                type(types).__name__
            )
        )
    if not isinstance(item, types):
        if isinstance(types, tuple):
            ", ".join(types)
        raise AttributeError(
            "invalid type '{}' in section '{}', must be: {}".format(
                type(item).__name__,
                section,
                type_names
            )
        )


def convert_host_list_to_dict(inv):
    '''
    Iterate over the inventory data structure and convert any host groups that
    are lists into dictionary form
    '''
    for group in inv:
        if isinstance(inv[group], list):
            # needs converting
            inv[group] = {'hosts': inv[group]}


class Inventory(object):
    '''Retrieve Ansible inventory from available sources and return as JSON'''
    def __init__(self, uri, env=None, vault_password=None):
        self.session = requests.Session()
        self.config = fetch_data(uri, self.session)
        self.uri = uri
        self.env = env
        self.vault_password = <PASSWORD>_password

    def fetch_environments(self):
        '''Fetch a list of environments that are defined upstream'''
        return list(self.config.get('environments', {}).keys())

    def fetch(self, uri, data_type=None):
        '''fetch the requested uri'''
        uriobj = urlparse(uri)
        # Check if it's an absolute uri, or a relative
        if uriobj.scheme == '':
            # Unspecified URI type, assume relative to config URI
            return fetch_data(
                urljoin(str(self.uri), uriobj.path),
                self.session,
                data_type,
                self.vault_password
            )
        elif uriobj.scheme in ['file', 'http', 'https']:
            # supported URI types
            if uriobj.path.startswith('/'):
                # Absolute path, fetch it
                return fetch_data(uri, self.session, data_type, self.vault_password)
            else:
                # Assume relative to config URI
                return fetch_data(
                    urljoin(self.uri, uriobj.path),
                    self.session,
                    data_type,
                    self.vault_password
                )
        else:
            # Unsupported type
            raise AttributeError("Unsupported type '{}'".format(uriobj.scheme))


    def generate_inventory(self):
        '''Generate inventory by merging hosts and variables'''
        # Set the basic structure
        invdata = {
            '_meta': {
                'hostvars': {}
            },
            'all': {
                'vars': {
                    'platform_name': self.env
                }
            }
        }
        # start merging
        for inc in self.config.get('environments', {}).get(self.env, {}).get('include', []):
            convert_host_list_to_dict(invdata)
            if isinstance(inc, str):
                # Just a plain file listed with no extra properties
                # Pull it in and hope there are proper sections
                # TODO: add some schema validation
                from_file = self.fetch(inc)
                dpath.util.merge(invdata, from_file, flags=dpath.util.MERGE_REPLACE)
            elif isinstance(inc, dict):
                # Dictionary listing, fetch file in `path` into keyspace `key`
                from_file = self.fetch(inc['path'], inc.get('format'))
                if 'key' in inc:
                    # A key space is specified, load the file into it
                    key = inc['key']
                    try:
                        data = dpath.util.get(invdata, key)
                        dpath.util.merge(data, from_file, flags=dpath.util.MERGE_REPLACE)
                    except KeyError:
                        dpath.util.new(invdata, key, from_file)
                else:
                    # No keyspace defined, load the file into the root
                    dpath.util.merge(invdata, from_file, flags=dpath.util.MERGE_REPLACE)
            else:
                raise_for_type(inc, (str, dict), ':'.join(['', self.env, 'include']))

        return invdata


@click.command()
@click.option(
    '--env', default=get_environment(), envvar='INVENTORY_ENV', show_default=True,
    help='specify the platform name to pull inventory for'
)
@click.option(
    '--uri', envvar='INVENTORY_URI', show_default=True,
    default=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'etc', 'config.yaml'),
    help='specify the URI to query for inventory config file, supports file:// and http(s)://'
)
@click.option(
    '--vault-password-file', 'vpfile', show_default=True,
    type=click.Path(exists=True, dir_okay=False, resolve_path=True),
    envvar='VAULT_PASSWORD_FILE', default='~/.vault_pass.<PASSWORD>',
    help='vault password file, if set to /dev/null secret decryption will be disabled'
)
@click.option('--list', is_flag=True, help='Print inventory information as a JSON object')
@click.option('--host', help='Retrieve host variables (not implemented)')
@click.option(
    '--createlinks', 'linkdir',
    type=click.Path(exists=True, file_okay=False, writable=True),
    help='Create symlinks in DIRECTORY to the script for each platform name retrieved'
)
@click.option('--show', is_flag=True, help='Output a list of upstream environments')
@click.option(
    '--tree', is_flag=True,
    help='Output a tree of what files will be loaded for an environment'
)
def cli(env, uri, vpfile, list, host, linkdir, show, tree):
    '''Ansible file based dynamic inventory script'''

    # If vault password file is /dev/null, disable secrets decryption
    if vpfile == '/dev/null':
        vpfile = None

    # Read in the vault password if one was provided
    if vpfile is not None:
        vault_password = fetch_data_local(vpfile)
    else:
        vault_password = None

    inv = Inventory(uri, env, vault_password)

    # Called with `--createlinks`
    if linkdir:
        create_links(inv.fetch_environments(), linkdir)

    # Called with `--show`
    elif show:
        click.echo("Upstream environments:")
        click.echo("\n".join(sorted(inv.fetch_environments())))

    # Called with `--tree`
    elif tree:
        if env is None:
            click.echo(yaml.dump(inv.config.get('environments', {}), default_flow_style=False))
        else:
            click.echo(yaml.dump(
                inv.config.get('environments', {}).get(env),
                default_flow_style=False
            ))

    else:
        if env is None:
            click.echo("Error: Missing environment, use --env or `export INVENTORY_ENV`")
            return 1
        else:
            data = None

            # Called with `--list`.
            if list:
                data = inv.generate_inventory()

            # Called with `--host [hostname]`.
            elif host:
                # Not implemented, since we should return _meta info in `--list`.
                data = {}

            # This should never happen since argparse should require either --list or --host
            else:
                data = {"_meta": {"hostvars": {}}}

            click.echo(json.dumps(data))<commit_msg>Fix default config and return codes<commit_after>'''
Custom dynamic inventory script for Ansible, in Python.

This script will read in a configuration file either locally or fetched via HTTP and will
output a JSON data structure describing the inventory by merging the files as listed in
the config file.

Files can be in either YAML or JSON format
'''

# pylint: disable=wrong-import-order
# Support python 2 and 3
from __future__ import (absolute_import)
from future import standard_library
standard_library.install_aliases()
from builtins import object # pylint: disable=redefined-builtin

# stdlib
import json
import os
import sys

## This is the python3 name, formerly urlparse, install_aliases() above make
## this work under python2
from urllib.parse import (urlparse, urljoin) # pylint: disable=import-error

# extras from packages
import click
import dpath.util
import requests
import yaml

# Ansible stuff for Secrets
import ansible.parsing.vault as vault
from ansible.module_utils._text import to_text


def get_config():
    '''Determine location of root config file if none specified'''
    self_path = os.path.dirname(os.path.realpath(sys.argv[0]))
    check_paths = [
        os.path.abspath(os.path.join(self_path, '..', 'etc', 'config.yaml')),
        '/etc/aggravator/config.yaml',
        '/usr/local/etc/aggravator/config.yaml'
    ]
    for fp in check_paths:
        if os.path.isfile(fp):
            return fp
    return None


def get_environment():
    '''Determine the platform/environment name from name of called script'''
    myname = os.path.basename(sys.argv[0])
    if myname != 'inventory':
        return myname
    else:
        return None


def create_links(environments, directory):
    '''Create symlinks for platform inventories'''
    errcount = 0
    for ename in environments:
        try:
            os.symlink(
                os.path.relpath(__file__, directory),
                os.path.join(directory, ename)
            )
        except(OSError) as err:
            click.echo("This symlink might already exist. Leaving it unchanged. Error: %s" % (err))
            errcount += 1
    return errcount


def fetch_data_remote(url, requestsobj=requests):
    '''fetch data from url and return as plaintext'''
    response = requestsobj.get(url)
    if response.status_code == 404:
        raise LookupError("Failed to find data at: {}".format(url))
    response.raise_for_status()
    return response.text


def fetch_data_local(localfile):
    '''Fetch data from local file and return as plaintext'''
    this_path = os.path.realpath(os.path.expanduser(localfile))
    if not os.path.exists(this_path):
        raise IOError("The file %s was not found" % this_path)
    try:
        f_obj = open(this_path, "rb")
        data = f_obj.read().strip()
        f_obj.close()
    except (OSError, IOError) as err:
        raise LookupError("Could not read file %s: %s" % (this_path, err))
    return to_text(data, errors='surrogate_or_strict')


def fetch_data(uri, requestsobj=requests, data_type=None, vault_password=None):
    '''Fetch data using either local or remote functions'''
    uriobj = urlparse(uri)
    loader = {
        'json': getattr(json, 'loads'),
        'yaml': getattr(yaml, 'load')
    }
    if data_type is None:
        # guess the data type from file extension
        if uriobj.path.endswith(('.yaml', '.yml')):
            data_type = 'yaml'
        else:
            data_type = os.path.splitext(uriobj.path)[1]
    data_type = data_type.lower()
    if data_type not in loader:
        raise AttributeError("Unsupported data type: {}".format(data_type))
    parser = loader[data_type]
    if uriobj.scheme in ['file', '']:
        data = fetch_data_local(uriobj.path)
    elif uriobj.scheme in ['http', 'https']:
        data = fetch_data_remote(uri, requestsobj)
    else:
        raise AttributeError("unsupported URI '{}'".format(uri))
    if vault.is_encrypted(data):
        if vault_password is None:
            data = '{}'
        else:
            data = vault.VaultLib(vault_password).decrypt(data)
    return parser(data)


def raise_for_type(item, types, section):
    '''raise an AttributeError if `item` is not of `types`'''
    type_names = None
    if isinstance(types, tuple):
        type_names = [t.__name__ for t in types]
    elif isinstance(types, type):
        type_names = types.__name__
    else:
        raise AttributeError(
            "Invalid type '{}' for `types` parameter, must be type or tuple of types".format(
                type(types).__name__
            )
        )
    if not isinstance(item, types):
        if isinstance(types, tuple):
            ", ".join(types)
        raise AttributeError(
            "invalid type '{}' in section '{}', must be: {}".format(
                type(item).__name__,
                section,
                type_names
            )
        )


def convert_host_list_to_dict(inv):
    '''
    Iterate over the inventory data structure and convert any host groups that
    are lists into dictionary form
    '''
    for group in inv:
        if isinstance(inv[group], list):
            # needs converting
            inv[group] = {'hosts': inv[group]}


class Inventory(object):
    '''Retrieve Ansible inventory from available sources and return as JSON'''
    def __init__(self, uri, env=None, vault_password=None):
        self.session = requests.Session()
        self.config = fetch_data(uri, self.session)
        self.uri = uri
        self.env = env
        self.vault_password = <PASSWORD>

    def fetch_environments(self):
        '''Fetch a list of environments that are defined upstream'''
        return list(self.config.get('environments', {}).keys())

    def fetch(self, uri, data_type=None):
        '''fetch the requested uri'''
        uriobj = urlparse(uri)
        # Check if it's an absolute uri, or a relative
        if uriobj.scheme == '':
            # Unspecified URI type, assume relative to config URI
            return fetch_data(
                urljoin(str(self.uri), uriobj.path),
                self.session,
                data_type,
                self.vault_password
            )
        elif uriobj.scheme in ['file', 'http', 'https']:
            # supported URI types
            if uriobj.path.startswith('/'):
                # Absolute path, fetch it
                return fetch_data(uri, self.session, data_type, self.vault_password)
            else:
                # Assume relative to config URI
                return fetch_data(
                    urljoin(self.uri, uriobj.path),
                    self.session,
                    data_type,
                    self.vault_password
                )
        else:
            # Unsupported type
            raise AttributeError("Unsupported type '{}'".format(uriobj.scheme))


    def generate_inventory(self):
        '''Generate inventory by merging hosts and variables'''
        # Set the basic structure
        invdata = {
            '_meta': {
                'hostvars': {}
            },
            'all': {
                'vars': {
                    'platform_name': self.env
                }
            }
        }
        # start merging
        for inc in self.config.get('environments', {}).get(self.env, {}).get('include', []):
            convert_host_list_to_dict(invdata)
            if isinstance(inc, str):
                # Just a plain file listed with no extra properties
                # Pull it in and hope there are proper sections
                # TODO: add some schema validation
                from_file = self.fetch(inc)
                dpath.util.merge(invdata, from_file, flags=dpath.util.MERGE_REPLACE)
            elif isinstance(inc, dict):
                # Dictionary listing, fetch file in `path` into keyspace `key`
                from_file = self.fetch(inc['path'], inc.get('format'))
                if 'key' in inc:
                    # A key space is specified, load the file into it
                    key = inc['key']
                    try:
                        data = dpath.util.get(invdata, key)
                        dpath.util.merge(data, from_file, flags=dpath.util.MERGE_REPLACE)
                    except KeyError:
                        dpath.util.new(invdata, key, from_file)
                else:
                    # No keyspace defined, load the file into the root
                    dpath.util.merge(invdata, from_file, flags=dpath.util.MERGE_REPLACE)
            else:
                raise_for_type(inc, (str, dict), ':'.join(['', self.env, 'include']))

        return invdata


@click.command()
@click.option(
    '--env', default=get_environment(), envvar='INVENTORY_ENV', show_default=True,
    help='specify the platform name to pull inventory for'
)
@click.option(
    '--uri', envvar='INVENTORY_URI', show_default=True,
    default=get_config(),
    help='specify the URI to query for inventory config file, supports file:// and http(s)://'
)
@click.option(
    '--vault-password-file', 'vpfile', show_default=True,
    type=click.Path(exists=True, dir_okay=False, resolve_path=True),
    envvar='VAULT_PASSWORD_FILE', default='~/.vault_pass.<PASSWORD>',
    help='vault password file, if set to /dev/null secret decryption will be disabled'
)
@click.option('--list', is_flag=True, help='Print inventory information as a JSON object')
@click.option('--host', help='Retrieve host variables (not implemented)')
@click.option(
    '--createlinks', 'linkdir',
    type=click.Path(exists=True, file_okay=False, writable=True),
    help='Create symlinks in DIRECTORY to the script for each platform name retrieved'
)
@click.option('--show', is_flag=True, help='Output a list of upstream environments')
@click.option(
    '--tree', is_flag=True,
    help='Output a tree of what files will be loaded for an environment'
)
def cli(env, uri, vpfile, list, host, linkdir, show, tree):
    '''Ansible file based dynamic inventory script'''

    # If vault password file is /dev/null, disable secrets decryption
    if vpfile == '/dev/null':
        vpfile = None

    # Read in the vault password if one was provided
    if vpfile is not None:
        vault_password = fetch_data_local(vpfile)
    else:
        vault_password = None

    inv = Inventory(uri, env, vault_password)

    # Called with `--createlinks`
    if linkdir:
        return create_links(inv.fetch_environments(), linkdir)

    # Called with `--show`
    elif show:
        click.echo("Upstream environments:")
        click.echo("\n".join(sorted(inv.fetch_environments())))
        return 0

    # Called with `--tree`
    elif tree:
        data = ''
        if env is None:
            data = yaml.dump(inv.config.get('environments', {}), default_flow_style=False)
        else:
            data = yaml.dump(
                inv.config.get('environments', {}).get(env),
                default_flow_style=False
            )
        click.echo(data)
        return 0

    else:
        if env is None:
            click.echo("Error: Missing environment, use --env or `export INVENTORY_ENV`")
            return 1
        else:
            data = None

            # Called with `--list`.
            if list:
                data = inv.generate_inventory()

            # Called with `--host [hostname]`.
            elif host:
                # Not implemented, since we should return _meta info in `--list`.
                data = {}

            # require either --list or --host
            else:
                click.echo("Error: Missing parameter (--list or --host)?")
                return 1

            click.echo(json.dumps(data))
            return 0
{'fraction_non_alphanumeric': 0.10762960690652526, 'fraction_numerical': 0.001183224505894211, 'mean_word_length': 2.2359614293817356, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3476472', 'n_tokens_mistral': 6568, 'n_tokens_neox': 6333, 'n_words': 1906}