<commit_before># pyinfra
# File: pyinfra/cli/inventory.py
# Desc: inventory creation for the CLI

from fnmatch import fnmatch
from os import path
from types import GeneratorType

import six

from pyinfra import logger, pseudo_inventory
from pyinfra.api.inventory import Inventory
from pyinfra.api.util import exec_file
from . import CliError


# Hosts in an inventory can be just the hostname or a tuple (hostname, data)
ALLOWED_HOST_TYPES = tuple(
    list(six.string_types) + [tuple],
)

# Group data can be any "core" Python type
ALLOWED_DATA_TYPES = tuple(
    list(six.string_types) + list(six.integer_types)
    + [bool, dict, list, set, tuple, float, complex],
)


def is_inventory_group(key, value):
    '''
    Verify that a module-level variable (key = value) is a valid inventory group.
    '''

    if (
        key.startswith('_')
        or not isinstance(value, (list, tuple, GeneratorType))
    ):
        return False

    # If the group is a tuple of (hosts, data), check the hosts
    if isinstance(value, tuple):
        value = value[0]

    # Expand any generators of hosts
    if isinstance(value, GeneratorType):
        value = list(value)

    return all(
        isinstance(item, ALLOWED_HOST_TYPES)
        for item in value
    )


def is_group_data(key, value):
    '''
    Verify that a module-level variable (key = value) is a valid bit of group data.
    '''

    return (
        isinstance(value, ALLOWED_DATA_TYPES)
        and not key.startswith('_')
    )


def make_inventory(
    inventory_filename,
    deploy_dir=None, limit=None,
    ssh_user=None, ssh_key=None, ssh_key_password=<PASSWORD>,
    ssh_port=None, ssh_password=<PASSWORD>,
):
    '''
    Builds a ``pyinfra.api.Inventory`` from the filesystem. If the file does not exist
    and doesn't contain a / attempts to use that as the only hostname.
    '''

    if ssh_port is not None:
        ssh_port = int(ssh_port)

    file_groupname = None

    try:
        attrs = exec_file(inventory_filename, return_locals=True)

        groups = {
            key: value
            for key, value in six.iteritems(attrs)
            if is_inventory_group(key, value)
        }

        # Used to set all the hosts to an additional group - that of the filename
        # ie inventories/dev.py means all the hosts are in the dev group, if not present
        file_groupname = path.basename(inventory_filename).split('.')[0]

    except IOError as e:
        # If a /, definitely not a hostname
        if '/' in inventory_filename:
            raise CliError('{0}: {1}'.format(e.strerror, inventory_filename))

        # Otherwise we assume the inventory is actually a hostname or list of hostnames
        groups = {
            'all': inventory_filename.split(','),
        }

    all_data = {}

    if 'all' in groups:
        all_hosts = groups.pop('all')

        if isinstance(all_hosts, tuple):
            all_hosts, all_data = all_hosts

    # Build all out of the existing hosts if not defined
    else:
        all_hosts = []
        for hosts in groups.values():
            # Groups can be a list of hosts or tuple of (hosts, data)
            hosts = hosts[0] if isinstance(hosts, tuple) else hosts

            for host in hosts:
                # Hosts can be a hostname or tuple of (hostname, data)
                hostname = host[0] if isinstance(host, tuple) else host
<commit_msg>Tweak the error message for missing inventory.<commit_after># pyinfra
# File: pyinfra/cli/inventory.py
# Desc: inventory creation for the CLI

from fnmatch import fnmatch
from os import path
from types import GeneratorType

import six

from pyinfra import logger, pseudo_inventory
from pyinfra.api.inventory import Inventory
from pyinfra.api.util import exec_file
from . import CliError


# Hosts in an inventory can be just the hostname or a tuple (hostname, data)
ALLOWED_HOST_TYPES = tuple(
    list(six.string_types) + [tuple],
)

# Group data can be any "core" Python type
ALLOWED_DATA_TYPES = tuple(
    list(six.string_types) + list(six.integer_types)
    + [bool, dict, list, set, tuple, float, complex],
)


def is_inventory_group(key, value):
    '''
    Verify that a module-level variable (key = value) is a valid inventory group.
    '''

    if (
        key.startswith('_')
        or not isinstance(value, (list, tuple, GeneratorType))
    ):
        return False

    # If the group is a tuple of (hosts, data), check the hosts
    if isinstance(value, tuple):
        value = value[0]

    # Expand any generators of hosts
    if isinstance(value, GeneratorType):
        value = list(value)

    return all(
        isinstance(item, ALLOWED_HOST_TYPES)
        for item in value
    )


def is_group_data(key, value):
    '''
    Verify that a module-level variable (key = value) is a valid bit of group data.
    '''

    return (
        isinstance(value, ALLOWED_DATA_TYPES)
        and not key.startswith('_')
    )


def make_inventory(
    inventory_filename,
    deploy_dir=None, limit=None,
    ssh_user=None, ssh_key=None, ssh_key_password=<PASSWORD>,
    ssh_port=None, ssh_password=<PASSWORD>,
):
    '''
    Builds a ``pyinfra.api.Inventory`` from the filesystem. If the file does not exist
    and doesn't contain a / attempts to use that as the only hostname.
    '''

    if ssh_port is not None:
        ssh_port = int(ssh_port)

    file_groupname = None

    try:
        attrs = exec_file(inventory_filename, return_locals=True)

        groups = {
            key: value
            for key, value in six.iteritems(attrs)
            if is_inventory_group(key, value)
        }

        # Used to set all the hosts to an additional group - that of the filename
        # ie inventories/dev.py means all the hosts are in the dev group, if not present
        file_groupname = path.basename(inventory_filename).split('.')[0]

    except IOError:
        # If a /, definitely not a hostname
        if '/' in inventory_filename:
            raise CliError('No inventory file: {0}'.format(inventory_filename))

        # Otherwise we assume the inventory is actually a hostname or list of hostnames
        groups = {
            'all': inventory_filename.split(','),
        }

    all_data = {}

    if 'all' in groups:
        all_hosts = groups.pop('all')

        if isinstance(all_hosts, tuple):
            all_hosts, all_data = all_hosts

    # Build all out of the existing hosts if not defined
    else:
        all_hosts = []
        for hosts in groups.values():
            # Groups can be a list of hosts or tuple of (hosts, data)
            hosts = hosts[0] if isinstance(hosts, tuple) else hosts

            for host in hosts:
                # Hosts can be a hostname or tuple of (hostname, data)
                hostname = host[0] if isinstance(host, tuple) else host

{'fraction_non_alphanumeric': 0.0864143192488263, 'fraction_numerical': 0.001613849765258216, 'mean_word_length': 2.5616509926854754, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5737006', 'n_tokens_mistral': 2087, 'n_tokens_neox': 2004, 'n_words': 705}