<commit_before># pyinfra
# File: pyinfra/api/state.py
# Desc: class that represents the current pyinfra.state

from __future__ import division, unicode_literals

from uuid import uuid4

from gevent.pool import Pool

from .config import Config
from .util import sha1_hash
from .exceptions import PyinfraError
from .pipelining import PipelineFacts


class State(object):
    '''
    Manages state for a pyinfra deploy.
    '''

    # A pyinfra.api.Inventory which stores all our pyinfra.api.Host's
    inventory = None

    # A pyinfra.api.Config
    config = None

    # Main gevent pool
    pool = None

    # Whether we are in an @operation (so inner ops aren't wrapped)
    in_op = False

    # Current op args tuple (sudo, sudo_user, ignore_errors) for use w/facts
    current_op_meta = None

    # Flag for pipelining mode
    pipelining = False

    # List of hosts to only apply operations to
    limit_hosts = None

    # Flags for printing
    print_output = False  # print output from the actual deploy (-v)
    print_fact_info = False  # log fact gathering as INFO > DEBUG (-v)
    print_fact_output = False  # print output from facts (-vv)
    print_lines = False  # print blank lines between operations (always in CLI)

    # Used in CLI
    deploy_dir = None  # base directory for locating files/templates/etc
    active = True  # used to disable operation calls when scanning deploy.py for config

    def __init__(self, inventory, config=None):
        # Connection storage
        self.ssh_connections = {}
        self.sftp_connections = {}

        # Facts storage
        self.facts = {}
        self.fact_locks = {}

        # If no config, create one using the defaults
        if config is None:
            config = Config()

        if not config.PARALLEL:
            config.PARALLEL = len(inventory)<commit_msg>Make more educated guess at parallel default based on nofile limits.<commit_after># pyinfra
# File: pyinfra/api/state.py
# Desc: class that represents the current pyinfra.state

from __future__ import division, unicode_literals

from uuid import uuid4
from resource import getrlimit, RLIMIT_NOFILE

from gevent.pool import Pool

from pyinfra import logger

from .config import Config
from .util import sha1_hash
from .exceptions import PyinfraError
from .pipelining import PipelineFacts


# Work out the max parallel we can achieve with the open files limit of the user/process,
# take 10 for opening Python files and /3 for ~3 files per host during op runs.
# See: https://github.com/Fizzadar/pyinfra/issues/44
nofile_limit, _ = getrlimit(RLIMIT_NOFILE)
MAX_PARALLEL = round((nofile_limit - 10) / 3)


class State(object):
    '''
    Manages state for a pyinfra deploy.
    '''

    # A pyinfra.api.Inventory which stores all our pyinfra.api.Host's
    inventory = None

    # A pyinfra.api.Config
    config = None

    # Main gevent pool
    pool = None

    # Whether we are in an @operation (so inner ops aren't wrapped)
    in_op = False

    # Current op args tuple (sudo, sudo_user, ignore_errors) for use w/facts
    current_op_meta = None

    # Flag for pipelining mode
    pipelining = False

    # List of hosts to only apply operations to
    limit_hosts = None

    # Flags for printing
    print_output = False  # print output from the actual deploy (-v)
    print_fact_info = False  # log fact gathering as INFO > DEBUG (-v)
    print_fact_output = False  # print output from facts (-vv)
    print_lines = False  # print blank lines between operations (always in CLI)

    # Used in CLI
    deploy_dir = None  # base directory for locating files/templates/etc
    active = True  # used to disable operation calls when scanning deploy.py for config

    def __init__(self, inventory, config=None):
        # Connection storage
        self.ssh_connections = {}
        self.sftp_connections = {}

        # Facts storage
        self.facts = {}
        self.fact_locks = {}

        # If no config, create one using the defaults
        if config is None:
            config = Config()

        if not config.PARALLEL:
            # If possible run everything in parallel, otherwise the max as defined above
            config.PARALLEL = min(len(inventory), MAX_PARALLEL)

        else:
            # If explicitly set, just issue a warning
            if config.PARALLEL > MAX_PARALLEL:
                logger.warning((
                    'Parallel set to {0}, but this may hit the open files limit of {1}'
                ).format(config.PARALLEL, nofile_limit))
{'fraction_non_alphanumeric': 0.06727152024784244, 'fraction_numerical': 0.0033193184332816995, 'mean_word_length': 2.695829926410466, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '471033', 'n_tokens_mistral': 1342, 'n_tokens_neox': 1256, 'n_words': 495}