<commit_before># Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.

# Python
import json
import logging
from optparse import make_option
import os
import re
import subprocess
import sys
import time
import traceback
import shutil

# Django
from django.conf import settings
from django.core.management.base import NoArgsCommand, CommandError
from django.core.exceptions import ImproperlyConfigured
from django.db import connection, transaction
from django.utils.encoding import smart_text

# AWX
from awx.main.models import * # noqa
from awx.main.task_engine import TaskEnhancer
from awx.main.utils import (
    ignore_inventory_computed_fields,
    check_proot_installed,
    wrap_args_with_proot,
    build_proot_temp_dir
)
from awx.main.utils.mem_inventory import MemInventory, dict_to_mem_data
from awx.main.signals import disable_activity_stream

logger = logging.getLogger('awx.main.commands.inventory_import')

LICENSE_EXPIRED_MESSAGE = '''\
License expired.
See http://www.ansible.com/renew for license extension information.'''

LICENSE_NON_EXISTANT_MESSAGE = '''\
No license.
See http://www.ansible.com/renew for license information.'''

LICENSE_MESSAGE = '''\
Number of licensed instances exceeded, would bring available instances to %(new_count)d, system is licensed for %(available_instances)d.
See http://www.ansible.com/renew for license extension information.'''

DEMO_LICENSE_MESSAGE = '''\
Demo mode free license count exceeded, would bring available instances to %(new_count)d, demo mode allows %(available_instances)d.
See http://www.ansible.com/renew for licensing information.'''


def functioning_dir(path):
    if os.path.isdir(path):
        return path
    return os.path.dirname(path)


class AnsibleInventoryLoader(object):
    '''
    Given executable `source` (directory, executable, or file) this will
    use the ansible-inventory CLI utility to convert it into in-memory
    representational objects. Example:
        /usr/bin/ansible/ansible-inventory -i hosts --list
    If it fails to find this, it uses the backported script instead
    '''

    def __init__(self, source, group_filter_re=None, host_filter_re=None, is_custom=False):
        self.source = source
        self.source_dir = functioning_dir(self.source)
        self.is_custom = is_custom
        self.tmp_private_dir = None
        self.method = 'ansible-inventory'
        self.group_filter_re = group_filter_re
        self.host_filter_re = host_filter_re

        self.is_vendored_source = False
        if self.source_dir == os.path.join(settings.BASE_DIR, 'plugins', 'inventory'):
            self.is_vendored_source = True

    def build_env(self):
        # Use ansible venv if it's available and setup to use
        env = dict(os.environ.items())
        if settings.ANSIBLE_USE_VENV:
            env['VIRTUAL_ENV'] = settings.ANSIBLE_VENV_PATH
            env['PATH'] = os.path.join(settings.ANSIBLE_VENV_PATH, "bin") + ":" + env['PATH']
            venv_libdir = os.path.join(settings.ANSIBLE_VENV_PATH, "lib")
            env.pop('PYTHONPATH', None)  # default to none if no python_ver matches
            for python_ver in ["python2.7", "python2.6"]:
                if os.path.isdir(os.path.join(venv_libdir, python_ver)):
                    env['PYTHONPATH'] = os.path.join(venv_libdir, python_ver, "site-packages") + ":"
                    break
        return env

    def get_base_args(self):
        # get ansible-inventory absolute path for running in bubblewrap/proot, in Popen
        for path in os.environ["PATH"].split(os.pathsep):
            potential_path = os.path.join(path.strip('"'), 'ansible-inventory')
            if os.path.isfile(potential_path) and os.access(potential_path, os.X_OK):
                logger.debug('Using system install of ansible-inventory CLI: {}'.format(potential_path))
                return [potential_path, '-i', self.source]

        # Stopgap solution for group_vars, do not use backported module for official
        # vendored cloud modules or custom scripts TODO: remove after Ansible 2.3 deprecation
        if self.is_vendored_source or self.is_custom:
            self.method = 'inventory script invocation'
            return [self.source]

        # ansible-inventory was not found, look for backported module TODO: remove after Ansible 2.3 deprecation
        abs_module_path = os.path.abspath(os.path.join(
            os.path.dirname(__file__), '..', '..', '..', 'plugins',
            'ansible_inventory', 'backport.py'))
        self.method = 'ansible-inventory backport'

        if not os.path.exists(abs_module_path):
            raise ImproperlyConfigured('Can not find inventory module')
        logger.debug('Using backported ansible-inventory module: {}'.format(abs_module_path))
        return [abs_module_path, '-i', self.source]

    def get_proot_args(self, cmd, env):
        cwd = os.getcwd()
        if not check_proot_installed():
            raise RuntimeError("proot is not installed but is configured for use")

        kwargs = {}
        if self.is_custom:
            # use source's tmp dir for proot, task manager will delete folder
            logger.debug("Using provided directory '{}' for isolation.".format(self.source_dir))
            kwargs['proot_temp_dir'] = self.source_dir
            cwd = self.source_dir
        else:
            # we can not safely store tmp data in source dir or trust script contents
            if env['AWX_PRIVATE_DATA_DIR']:
                # If this is non-blank, file credentials are being used and we need access
                private_data_dir = functioning_dir(env['AWX_PRIVATE_DATA_DIR'])
                logger.debug("Using private credential data in '{}'.".format(private_data_dir))
                kwargs['private_data_dir'] = private_data_dir
            self.tmp_private_dir = build_proot_temp_dir()
            logger.debug("Using fresh temporary directory '{}' for isolation.".format(self.tmp_private_dir))
            kwargs['proot_temp_dir'] = self.tmp_private_dir
            # Run from source's location so that custom script contents are in `show_paths`
            cwd = functioning_dir(self.source)
        logger.debug("Running from `{}` working directory.".format(cwd))

        return wrap_args_with_proot(cmd, cwd, **kwargs)

    def command_to_json(self, cmd):
        data = {}
        stdout, stderr = '', ''
        env = self.build_env()

        if ((self.is_custom or 'AWX_PRIVATE_DATA_DIR' in env) and
                getattr(settings, 'AWX_PROOT_ENABLED', False)):
            cmd = self.get_proot_args(cmd, env)

        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
        stdout, stderr = proc.communicate()

        if self.tmp_private_dir:
            shutil.rmtree(self.tmp_private_dir, True)
        if proc.returncode != 0 or 'file not found' in stderr:
            raise RuntimeError('%s failed (rc=%d) with stdout:\n%s\nstderr:\n%s' % (
                self.method, proc.returncode, stdout, stderr))

        try:
            data = json.loads(stdout)
            if not isinstance(data, dict):
                raise TypeError('Returned JSON must be a dictionary, got %s instead' % str(type(data)))
        except:
            logger.error('Failed to load JSON from: %s', stdout)
            raise
        return data

    def load(self):
        base_args = self.get_base_args()
        logger.info('Reading Ansible inventory source: %s', self.source)
        data = self.command_to_json(base_args + ['--list'])

        logger.info('Processing JSON output...')
        inventory = MemInventory(
            group_filter_re=self.group_filter_re, host_filter_re=self.host_filter_re)
        inventory = dict_to_mem_data(data, inventory=inventory)
<commit_msg>store stderr on successful inventory syncs<commit_after># Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.

# Python
import json
import logging
from optparse import make_option
import os
import re
import subprocess
import sys
import time
import traceback
import shutil

# Django
from django.conf import settings
from django.core.management.base import NoArgsCommand, CommandError
from django.core.exceptions import ImproperlyConfigured
from django.db import connection, transaction
from django.utils.encoding import smart_text

# AWX
from awx.main.models import * # noqa
from awx.main.task_engine import TaskEnhancer
from awx.main.utils import (
    ignore_inventory_computed_fields,
    check_proot_installed,
    wrap_args_with_proot,
    build_proot_temp_dir
)
from awx.main.utils.mem_inventory import MemInventory, dict_to_mem_data
from awx.main.signals import disable_activity_stream

logger = logging.getLogger('awx.main.commands.inventory_import')

LICENSE_EXPIRED_MESSAGE = '''\
License expired.
See http://www.ansible.com/renew for license extension information.'''

LICENSE_NON_EXISTANT_MESSAGE = '''\
No license.
See http://www.ansible.com/renew for license information.'''

LICENSE_MESSAGE = '''\
Number of licensed instances exceeded, would bring available instances to %(new_count)d, system is licensed for %(available_instances)d.
See http://www.ansible.com/renew for license extension information.'''

DEMO_LICENSE_MESSAGE = '''\
Demo mode free license count exceeded, would bring available instances to %(new_count)d, demo mode allows %(available_instances)d.
See http://www.ansible.com/renew for licensing information.'''


def functioning_dir(path):
    if os.path.isdir(path):
        return path
    return os.path.dirname(path)


class AnsibleInventoryLoader(object):
    '''
    Given executable `source` (directory, executable, or file) this will
    use the ansible-inventory CLI utility to convert it into in-memory
    representational objects. Example:
        /usr/bin/ansible/ansible-inventory -i hosts --list
    If it fails to find this, it uses the backported script instead
    '''

    def __init__(self, source, group_filter_re=None, host_filter_re=None, is_custom=False):
        self.source = source
        self.source_dir = functioning_dir(self.source)
        self.is_custom = is_custom
        self.tmp_private_dir = None
        self.method = 'ansible-inventory'
        self.group_filter_re = group_filter_re
        self.host_filter_re = host_filter_re

        self.is_vendored_source = False
        if self.source_dir == os.path.join(settings.BASE_DIR, 'plugins', 'inventory'):
            self.is_vendored_source = True

    def build_env(self):
        # Use ansible venv if it's available and setup to use
        env = dict(os.environ.items())
        if settings.ANSIBLE_USE_VENV:
            env['VIRTUAL_ENV'] = settings.ANSIBLE_VENV_PATH
            env['PATH'] = os.path.join(settings.ANSIBLE_VENV_PATH, "bin") + ":" + env['PATH']
            venv_libdir = os.path.join(settings.ANSIBLE_VENV_PATH, "lib")
            env.pop('PYTHONPATH', None)  # default to none if no python_ver matches
            for python_ver in ["python2.7", "python2.6"]:
                if os.path.isdir(os.path.join(venv_libdir, python_ver)):
                    env['PYTHONPATH'] = os.path.join(venv_libdir, python_ver, "site-packages") + ":"
                    break
        return env

    def get_base_args(self):
        # get ansible-inventory absolute path for running in bubblewrap/proot, in Popen
        for path in os.environ["PATH"].split(os.pathsep):
            potential_path = os.path.join(path.strip('"'), 'ansible-inventory')
            if os.path.isfile(potential_path) and os.access(potential_path, os.X_OK):
                logger.debug('Using system install of ansible-inventory CLI: {}'.format(potential_path))
                return [potential_path, '-i', self.source]

        # Stopgap solution for group_vars, do not use backported module for official
        # vendored cloud modules or custom scripts TODO: remove after Ansible 2.3 deprecation
        if self.is_vendored_source or self.is_custom:
            self.method = 'inventory script invocation'
            return [self.source]

        # ansible-inventory was not found, look for backported module TODO: remove after Ansible 2.3 deprecation
        abs_module_path = os.path.abspath(os.path.join(
            os.path.dirname(__file__), '..', '..', '..', 'plugins',
            'ansible_inventory', 'backport.py'))
        self.method = 'ansible-inventory backport'

        if not os.path.exists(abs_module_path):
            raise ImproperlyConfigured('Can not find inventory module')
        logger.debug('Using backported ansible-inventory module: {}'.format(abs_module_path))
        return [abs_module_path, '-i', self.source]

    def get_proot_args(self, cmd, env):
        cwd = os.getcwd()
        if not check_proot_installed():
            raise RuntimeError("proot is not installed but is configured for use")

        kwargs = {}
        if self.is_custom:
            # use source's tmp dir for proot, task manager will delete folder
            logger.debug("Using provided directory '{}' for isolation.".format(self.source_dir))
            kwargs['proot_temp_dir'] = self.source_dir
            cwd = self.source_dir
        else:
            # we can not safely store tmp data in source dir or trust script contents
            if env['AWX_PRIVATE_DATA_DIR']:
                # If this is non-blank, file credentials are being used and we need access
                private_data_dir = functioning_dir(env['AWX_PRIVATE_DATA_DIR'])
                logger.debug("Using private credential data in '{}'.".format(private_data_dir))
                kwargs['private_data_dir'] = private_data_dir
            self.tmp_private_dir = build_proot_temp_dir()
            logger.debug("Using fresh temporary directory '{}' for isolation.".format(self.tmp_private_dir))
            kwargs['proot_temp_dir'] = self.tmp_private_dir
            # Run from source's location so that custom script contents are in `show_paths`
            cwd = functioning_dir(self.source)
        logger.debug("Running from `{}` working directory.".format(cwd))

        return wrap_args_with_proot(cmd, cwd, **kwargs)

    def command_to_json(self, cmd):
        data = {}
        stdout, stderr = '', ''
        env = self.build_env()

        if ((self.is_custom or 'AWX_PRIVATE_DATA_DIR' in env) and
                getattr(settings, 'AWX_PROOT_ENABLED', False)):
            cmd = self.get_proot_args(cmd, env)

        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
        stdout, stderr = proc.communicate()

        if self.tmp_private_dir:
            shutil.rmtree(self.tmp_private_dir, True)
        if proc.returncode != 0 or 'file not found' in stderr:
            raise RuntimeError('%s failed (rc=%d) with stdout:\n%s\nstderr:\n%s' % (
                self.method, proc.returncode, stdout, stderr))

        for line in stderr.splitlines():
            logger.error(line)
        try:
            data = json.loads(stdout)
            if not isinstance(data, dict):
                raise TypeError('Returned JSON must be a dictionary, got %s instead' % str(type(data)))
        except:
            logger.error('Failed to load JSON from: %s', stdout)
            raise
        return data

    def load(self):
        base_args = self.get_base_args()
        logger.info('Reading Ansible inventory source: %s', self.source)
        data = self.command_to_json(base_args + ['--list'])

        logger.info('Processing JSON output...')
        inventory = MemInventory(
            group_filter_re=self.group_filter_re, host_filter_re=self.host_filter_re)
        inventory = dict_to_mem_data(data, inventory=inventory)

{'fraction_non_alphanumeric': 0.09880716973910825, 'fraction_numerical': 0.001658480576640939, 'mean_word_length': 2.906802890605532, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2395039', 'n_tokens_mistral': 4717, 'n_tokens_neox': 4552, 'n_words': 1204}