<commit_before># -*- coding: utf-8 -*-
'''
This module contains all of the routines needed to set up a master server, this
involves preparing the three listeners and the workers needed by the master.
'''
from __future__ import absolute_import

# Import python libs
import fnmatch
import logging
import os
import re
import time
import stat
import msgpack

# Import salt libs
import salt.crypt
import salt.utils
import salt.cache
import salt.client
import salt.payload
import salt.pillar
import salt.state
import salt.runner
import salt.auth
import salt.wheel
import salt.minion
import salt.search
import salt.key
import salt.fileserver
import salt.utils.args
import salt.utils.atomicfile
import salt.utils.event
import salt.utils.verify
import salt.utils.minions
import salt.utils.gzip_util
import salt.utils.jid
from salt.defaults import DEFAULT_TARGET_DELIM
from salt.pillar import git_pillar
from salt.utils.event import tagify
from salt.exceptions import FileserverConfigError, SaltMasterError

# Import 3rd-party libs
import salt.ext.six as six

try:
    import pwd
    HAS_PWD = True
except ImportError:
    # pwd is not available on windows
    HAS_PWD = False

log = logging.getLogger(__name__)

# Things to do in lower layers:
# only accept valid minion ids


def init_git_pillar(opts):
    '''
    Clear out the ext pillar caches, used when the master starts
    '''
    ret = []
    for opts_dict in [x for x in opts.get('ext_pillar', [])]:
        if 'git' in opts_dict:
            if isinstance(opts_dict['git'], six.string_types):
                # Legacy git pillar code
                try:
                    import git
                except ImportError:
                    return ret
                parts = opts_dict['git'].strip().split()
                try:
                    br = parts[0]
                    loc = parts[1]
                except IndexError:
                    log.critical(
                        'Unable to extract external pillar data: {0}'
                        .format(opts_dict['git'])
                    )
                else:
                    ret.append(
                        git_pillar._LegacyGitPillar(
                            br,
                            loc,
                            opts
                        )
                    )
            else:
                # New git_pillar code
                try:
                    pillar = salt.utils.gitfs.GitPillar(opts)
                    pillar.init_remotes(
                        opts_dict['git'],
                        git_pillar.PER_REMOTE_OVERRIDES,
                        git_pillar.PER_REMOTE_ONLY
                    )
                    ret.append(pillar)
                except FileserverConfigError:
                    if opts.get('git_pillar_verify_config', True):
                        raise
                    else:
                        log.critical('Could not initialize git_pillar')
    return ret


def clean_fsbackend(opts):
    '''
    Clean out the old fileserver backends
    '''
    # Clear remote fileserver backend caches so they get recreated
    for backend in ('git', 'hg', 'svn'):
        if backend in opts['fileserver_backend']:
            env_cache = os.path.join(
                opts['cachedir'],
                '{0}fs'.format(backend),
                'envs.p'
            )
            if os.path.isfile(env_cache):
                log.debug('Clearing {0}fs env cache'.format(backend))
                try:
                    os.remove(env_cache)
                except OSError as exc:
                    log.critical(
                        'Unable to clear env cache file {0}: {1}'
                        .format(env_cache, exc)
                    )

            file_lists_dir = os.path.join(
                opts['cachedir'],
                'file_lists',
                '{0}fs'.format(backend)
            )
            try:
                file_lists_caches = os.listdir(file_lists_dir)
            except OSError:
                continue
            for file_lists_cache in fnmatch.filter(file_lists_caches, '*.p'):
                cache_file = os.path.join(file_lists_dir, file_lists_cache)
                try:
                    os.remove(cache_file)
                except OSError as exc:
                    log.critical(
                        'Unable to file_lists cache file {0}: {1}'
                        .format(cache_file, exc)
                    )


def clean_expired_tokens(opts):
    '''
    Clean expired tokens from the master
    '''
    serializer = salt.payload.Serial(opts)
    for (dirpath, dirnames, filenames) in os.walk(opts['token_dir']):
        for token in filenames:
            token_path = os.path.join(dirpath, token)
            with salt.utils.fopen(token_path, 'rb') as token_file:
                try:
                    token_data = serializer.loads(token_file.read())
                except msgpack.UnpackValueError:
                    # Bad token file or empty. Remove.
                    os.remove(token_path)
                    return
                if 'expire' not in token_data or token_data.get('expire', 0) < time.time():
                    try:
                        os.remove(token_path)
                    except (IOError, OSError):
                        pass


def clean_pub_auth(opts):
    try:
        auth_cache = os.path.join(opts['cachedir'], 'publish_auth')
        if not os.path.exists(auth_cache):
            return
        else:
            for (dirpath, dirnames, filenames) in os.walk(auth_cache):
                for auth_file in filenames:
                    auth_file_path = os.path.join(dirpath, auth_file)
                    if not os.path.isfile(auth_file_path):
                        continue
                    if (time.time() - os.path.getmtime(auth_file_path) >
                            (opts['keep_jobs'] * 3600)):
                        os.remove(auth_file_path)
    except (IOError, OSError):
        log.error('Unable to delete pub auth file')


def clean_old_jobs(opts):
    '''
    Clean out the old jobs from the job cache
    '''
    # TODO: better way to not require creating the masterminion every time?
    mminion = salt.minion.MasterMinion(
                opts,
                states=False,
                rend=False,
                )
    # If the master job cache has a clean_old_jobs, call it
    fstr = '{0}.clean_old_jobs'.format(opts['master_job_cache'])
    if fstr in mminion.returners:
        mminion.returners[fstr]()


def mk_key(opts, user):
    if HAS_PWD:
        uid = None
        try:
            uid = pwd.getpwnam(user).pw_uid
        except KeyError:
            # User doesn't exist in the system
            if opts['client_acl_verify']:
                return None
    if salt.utils.is_windows():
        # The username may contain '\' if it is in Windows
        # 'DOMAIN\username' format. Fix this for the keyfile path.
        keyfile = os.path.join(
            opts['cachedir'], '.{0}_key'.format(user.replace('\\', '_'))
        )
    else:
        keyfile = os.path.join(
            opts['cachedir'], '.{0}_key'.format(user)
        )

    if os.path.exists(keyfile):
        log.debug('Removing stale keyfile: {0}'.format(keyfile))
        if salt.utils.is_windows() and not os.access(keyfile, os.W_OK):
            # Cannot delete read-only files on Windows.
            os.chmod(keyfile, stat.S_IRUSR | stat.S_IWUSR)
        os.unlink(keyfile)

    key = salt.crypt.Crypticle.generate_key_string()
    cumask = os.umask(191)
    with salt.utils.fopen(keyfile, 'w+') as fp_:
        fp_.write(key)
    os.umask(cumask)
    # 600 octal: Read and write access to the owner only.
    # Write access is necessary since on subsequent runs, if the file
    # exists, it needs to be written to again. Windows enforces this.
    os.chmod(keyfile, 0o600)
    if HAS_PWD and uid is not None:
        try:
            os.chown(keyfile, uid, -1)
        except OSError:
            # The master is not being run as root and can therefore not
            # chown the key file
            pass
    return key


def access_keys(opts):
    '''
    A key needs to be placed in the filesystem with permissions 0400 so
    clients are required to run as root.
    '''
    # TODO: Need a way to get all available users for systems not supported by pwd module.
    #       For now users pattern matching will not work for publisher_acl.
    keys = {}
    publisher_acl = opts['publisher_acl']
    acl_users = set(publisher_acl.keys())
    if opts.get('user'):
        acl_users.add(opts['user'])
    acl_users.add(salt.utils.get_user())
    for user in acl_users:
        log.info('Preparing the %s key for local communication', user)
        key = mk_key(opts, user)
        if key is not None:
            keys[user] = key

    # Check other users matching ACL patterns
    if opts['client_acl_verify'] and HAS_PWD:
        log.profile('Beginning pwd.getpwall() call in masterarpi access_keys function')
        for user in pwd.getpwall():
            user = user.pw_name
            if user not in keys and salt.utils.check_whitelist_blacklist(user, whitelist=acl_users):
                keys[user] = mk_key(opts, user)
        log.profile('End pwd.getpwall() call in masterarpi access_keys function')

    return keys


def fileserver_update(fileserver):
    '''
    Update the fileserver backends, requires that a built fileserver object
    be passed in
    '''
    try:
        if not fileserver.servers:<commit_msg>typo fix aka what is a 'masterarpi'<commit_after># -*- coding: utf-8 -*-
'''
This module contains all of the routines needed to set up a master server, this
involves preparing the three listeners and the workers needed by the master.
'''
from __future__ import absolute_import

# Import python libs
import fnmatch
import logging
import os
import re
import time
import stat
import msgpack

# Import salt libs
import salt.crypt
import salt.utils
import salt.cache
import salt.client
import salt.payload
import salt.pillar
import salt.state
import salt.runner
import salt.auth
import salt.wheel
import salt.minion
import salt.search
import salt.key
import salt.fileserver
import salt.utils.args
import salt.utils.atomicfile
import salt.utils.event
import salt.utils.verify
import salt.utils.minions
import salt.utils.gzip_util
import salt.utils.jid
from salt.defaults import DEFAULT_TARGET_DELIM
from salt.pillar import git_pillar
from salt.utils.event import tagify
from salt.exceptions import FileserverConfigError, SaltMasterError

# Import 3rd-party libs
import salt.ext.six as six

try:
    import pwd
    HAS_PWD = True
except ImportError:
    # pwd is not available on windows
    HAS_PWD = False

log = logging.getLogger(__name__)

# Things to do in lower layers:
# only accept valid minion ids


def init_git_pillar(opts):
    '''
    Clear out the ext pillar caches, used when the master starts
    '''
    ret = []
    for opts_dict in [x for x in opts.get('ext_pillar', [])]:
        if 'git' in opts_dict:
            if isinstance(opts_dict['git'], six.string_types):
                # Legacy git pillar code
                try:
                    import git
                except ImportError:
                    return ret
                parts = opts_dict['git'].strip().split()
                try:
                    br = parts[0]
                    loc = parts[1]
                except IndexError:
                    log.critical(
                        'Unable to extract external pillar data: {0}'
                        .format(opts_dict['git'])
                    )
                else:
                    ret.append(
                        git_pillar._LegacyGitPillar(
                            br,
                            loc,
                            opts
                        )
                    )
            else:
                # New git_pillar code
                try:
                    pillar = salt.utils.gitfs.GitPillar(opts)
                    pillar.init_remotes(
                        opts_dict['git'],
                        git_pillar.PER_REMOTE_OVERRIDES,
                        git_pillar.PER_REMOTE_ONLY
                    )
                    ret.append(pillar)
                except FileserverConfigError:
                    if opts.get('git_pillar_verify_config', True):
                        raise
                    else:
                        log.critical('Could not initialize git_pillar')
    return ret


def clean_fsbackend(opts):
    '''
    Clean out the old fileserver backends
    '''
    # Clear remote fileserver backend caches so they get recreated
    for backend in ('git', 'hg', 'svn'):
        if backend in opts['fileserver_backend']:
            env_cache = os.path.join(
                opts['cachedir'],
                '{0}fs'.format(backend),
                'envs.p'
            )
            if os.path.isfile(env_cache):
                log.debug('Clearing {0}fs env cache'.format(backend))
                try:
                    os.remove(env_cache)
                except OSError as exc:
                    log.critical(
                        'Unable to clear env cache file {0}: {1}'
                        .format(env_cache, exc)
                    )

            file_lists_dir = os.path.join(
                opts['cachedir'],
                'file_lists',
                '{0}fs'.format(backend)
            )
            try:
                file_lists_caches = os.listdir(file_lists_dir)
            except OSError:
                continue
            for file_lists_cache in fnmatch.filter(file_lists_caches, '*.p'):
                cache_file = os.path.join(file_lists_dir, file_lists_cache)
                try:
                    os.remove(cache_file)
                except OSError as exc:
                    log.critical(
                        'Unable to file_lists cache file {0}: {1}'
                        .format(cache_file, exc)
                    )


def clean_expired_tokens(opts):
    '''
    Clean expired tokens from the master
    '''
    serializer = salt.payload.Serial(opts)
    for (dirpath, dirnames, filenames) in os.walk(opts['token_dir']):
        for token in filenames:
            token_path = os.path.join(dirpath, token)
            with salt.utils.fopen(token_path, 'rb') as token_file:
                try:
                    token_data = serializer.loads(token_file.read())
                except msgpack.UnpackValueError:
                    # Bad token file or empty. Remove.
                    os.remove(token_path)
                    return
                if 'expire' not in token_data or token_data.get('expire', 0) < time.time():
                    try:
                        os.remove(token_path)
                    except (IOError, OSError):
                        pass


def clean_pub_auth(opts):
    try:
        auth_cache = os.path.join(opts['cachedir'], 'publish_auth')
        if not os.path.exists(auth_cache):
            return
        else:
            for (dirpath, dirnames, filenames) in os.walk(auth_cache):
                for auth_file in filenames:
                    auth_file_path = os.path.join(dirpath, auth_file)
                    if not os.path.isfile(auth_file_path):
                        continue
                    if (time.time() - os.path.getmtime(auth_file_path) >
                            (opts['keep_jobs'] * 3600)):
                        os.remove(auth_file_path)
    except (IOError, OSError):
        log.error('Unable to delete pub auth file')


def clean_old_jobs(opts):
    '''
    Clean out the old jobs from the job cache
    '''
    # TODO: better way to not require creating the masterminion every time?
    mminion = salt.minion.MasterMinion(
                opts,
                states=False,
                rend=False,
                )
    # If the master job cache has a clean_old_jobs, call it
    fstr = '{0}.clean_old_jobs'.format(opts['master_job_cache'])
    if fstr in mminion.returners:
        mminion.returners[fstr]()


def mk_key(opts, user):
    if HAS_PWD:
        uid = None
        try:
            uid = pwd.getpwnam(user).pw_uid
        except KeyError:
            # User doesn't exist in the system
            if opts['client_acl_verify']:
                return None
    if salt.utils.is_windows():
        # The username may contain '\' if it is in Windows
        # 'DOMAIN\username' format. Fix this for the keyfile path.
        keyfile = os.path.join(
            opts['cachedir'], '.{0}_key'.format(user.replace('\\', '_'))
        )
    else:
        keyfile = os.path.join(
            opts['cachedir'], '.{0}_key'.format(user)
        )

    if os.path.exists(keyfile):
        log.debug('Removing stale keyfile: {0}'.format(keyfile))
        if salt.utils.is_windows() and not os.access(keyfile, os.W_OK):
            # Cannot delete read-only files on Windows.
            os.chmod(keyfile, stat.S_IRUSR | stat.S_IWUSR)
        os.unlink(keyfile)

    key = salt.crypt.Crypticle.generate_key_string()
    cumask = os.umask(191)
    with salt.utils.fopen(keyfile, 'w+') as fp_:
        fp_.write(key)
    os.umask(cumask)
    # 600 octal: Read and write access to the owner only.
    # Write access is necessary since on subsequent runs, if the file
    # exists, it needs to be written to again. Windows enforces this.
    os.chmod(keyfile, 0o600)
    if HAS_PWD and uid is not None:
        try:
            os.chown(keyfile, uid, -1)
        except OSError:
            # The master is not being run as root and can therefore not
            # chown the key file
            pass
    return key


def access_keys(opts):
    '''
    A key needs to be placed in the filesystem with permissions 0400 so
    clients are required to run as root.
    '''
    # TODO: Need a way to get all available users for systems not supported by pwd module.
    #       For now users pattern matching will not work for publisher_acl.
    keys = {}
    publisher_acl = opts['publisher_acl']
    acl_users = set(publisher_acl.keys())
    if opts.get('user'):
        acl_users.add(opts['user'])
    acl_users.add(salt.utils.get_user())
    for user in acl_users:
        log.info('Preparing the %s key for local communication', user)
        key = mk_key(opts, user)
        if key is not None:
            keys[user] = key

    # Check other users matching ACL patterns
    if opts['client_acl_verify'] and HAS_PWD:
        log.profile('Beginning pwd.getpwall() call in masterapi access_keys function')
        for user in pwd.getpwall():
            user = user.pw_name
            if user not in keys and salt.utils.check_whitelist_blacklist(user, whitelist=acl_users):
                keys[user] = mk_key(opts, user)
        log.profile('End pwd.getpwall() call in masterapi access_keys function')

    return keys


def fileserver_update(fileserver):
    '''
    Update the fileserver backends, requires that a built fileserver object
    be passed in
    '''
    try:
        if not fileserver.servers:
{'fraction_non_alphanumeric': 0.08545131262420248, 'fraction_numerical': 0.0037652965171007216, 'mean_word_length': 1.6877020379479972, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '326033', 'n_tokens_mistral': 5599, 'n_tokens_neox': 5235, 'n_words': 1527}