<commit_before>'''
This module contains routines used to verify the matcher against the minions
expected to return
'''

# Import python libs
import os
import glob
import re
import logging

# Import salt libs
import salt.payload
import salt.utils

log = logging.getLogger(__name__)


def nodegroup_comp(group, nodegroups, skip=None):
    '''
    Take the nodegroup and the nodegroups and fill in nodegroup refs
    '''
    if skip is None:
        skip = set([group])
    if group not in nodegroups:
        return ''
    gstr = nodegroups[group]
    ret = ''
    for comp in gstr.split():
        if not comp.startswith('N@'):
            ret += '{0} '.format(comp)
            continue
        ngroup = comp[2:]
        if ngroup in skip:
            continue
        skip.add(ngroup)
        ret += nodegroup_comp(ngroup, nodegroups, skip)
    return ret


class CkMinions(object):
    '''
    Used to check what minions should respond from a target
    '''
    def __init__(self, opts):
        self.opts = opts
        self.serial = salt.payload.Serial(opts)

    def _check_glob_minions(self, expr):
        '''
        Return the minions found by looking via globs
        '''
        cwd = os.getcwd()
        os.chdir(os.path.join(self.opts['pki_dir'], 'minions'))
        ret = set(glob.glob(expr))
        os.chdir(cwd)
        return list(ret)

    def _check_list_minions(self, expr):
        '''
        Return the minions found by looking via a list
        '''
        ret = []
        for fn_ in os.listdir(os.path.join(self.opts['pki_dir'], 'minions')):
            if fn_ in expr:
                if fn_ not in ret:
                    ret.append(fn_)
        return ret

    def _check_pcre_minions(self, expr):
        '''
        Return the minions found by looking via regular expressions
        '''
        cwd = os.getcwd()
        os.chdir(os.path.join(self.opts['pki_dir'], 'minions'))
        reg = re.compile(expr)
        ret = [fn_ for fn_ in os.listdir('.') if reg.match(fn_)]
        os.chdir(cwd)
        return ret

    def _check_grain_minions(self, expr):
        '''
        Return the minions found by looking via grains
        '''
        minions = set(
            os.listdir(os.path.join(self.opts['pki_dir'], 'minions'))
        )
        if self.opts.get('minion_data_cache', False):
            cdir = os.path.join(self.opts['cachedir'], 'minions')
            if not os.path.isdir(cdir):
                return list(minions)
            for id_ in os.listdir(cdir):
                if id_ not in minions:
                    continue
                datap = os.path.join(cdir, id_, 'data.p')
                if not os.path.isfile(datap):
                    continue
                grains = self.serial.load(
                    salt.utils.fopen(datap)
                ).get('grains')
                if not salt.utils.subdict_match(grains, expr):
                    minions.remove(id_)
        return list(minions)

    def _check_grain_pcre_minions(self, expr):
        '''
        Return the minions found by looking via grains with PCRE
        '''
        minions = set(
            os.listdir(os.path.join(self.opts['pki_dir'], 'minions'))
        )
        if self.opts.get('minion_data_cache', False):
            cdir = os.path.join(self.opts['cachedir'], 'minions')
            if not os.path.isdir(cdir):
                return list(minions)
            for id_ in os.listdir(cdir):
                if id_ not in minions:
                    continue
                datap = os.path.join(cdir, id_, 'data.p')
                if not os.path.isfile(datap):
                    continue
                grains = self.serial.load(
                    salt.utils.fopen(datap)
                ).get('grains')
                if not salt.utils.subdict_match(grains, expr,
                                                delim=':', regex_match=True):
                    minions.remove(id_)
        return list(minions)

    def _check_pillar_minions(self, expr):
        '''
        Return the minions found by looking via pillar
        '''
        minions = set(
            os.listdir(os.path.join(self.opts['pki_dir'], 'minions'))
        )
        if self.opts.get('minion_data_cache', False):
            cdir = os.path.join(self.opts['cachedir'], 'minions')
            if not os.path.isdir(cdir):
                return list(minions)
            for id_ in os.listdir(cdir):
                if id_ not in minions:
                    continue
                datap = os.path.join(cdir, id_, 'data.p')
                if not os.path.isfile(datap):
                    continue
                pillar = self.serial.load(
                    salt.utils.fopen(datap)
                ).get('pillar')
                if not salt.utils.subdict_match(pillar, expr):
                    minions.remove(id_)
        return list(minions)

    def _check_ipcidr_minions(self, expr):
        '''
        Return the minions found by looking via ipcidr
        '''
        minions = set(
            os.listdir(os.path.join(self.opts['pki_dir'], 'minions'))
        )
        if self.opts.get('minion_data_cache', False):
            cdir = os.path.join(self.opts['cachedir'], 'minions')
            if not os.path.isdir(cdir):
                return list(minions)
            for id_ in os.listdir(cdir):
                if id_ not in minions:
                    continue
                datap = os.path.join(cdir, id_, 'data.p')
                if not os.path.isfile(datap):
                    continue
                grains = self.serial.load(
                    salt.utils.fopen(datap)
                ).get('grains')

                num_parts = len(expr.split('/'))
                if num_parts > 2:
                    # Target is not valid CIDR, no minions match
                    return []
                elif num_parts == 2:
                    # Target is CIDR
                    if not salt.utils.network.in_subnet(
                            expr,
                            addrs=grains.get('ipv4', [])):
                        minions.remove(id_)
                else:
                    # Target is an IPv4 address
                    import socket
                    try:
                        socket.inet_aton(expr)
                    except socket.error:
                        # Not a valid IPv4 address, no minions match
                        return []
                    else:
                        if not expr in grains.get('ipv4', []):
                            minions.remove(id_)
        return list(minions)

    def _check_compound_minions(self, expr):
        '''
        Return the minions found by looking via compound matcher
        '''
        minions = set(
            os.listdir(os.path.join(self.opts['pki_dir'], 'minions'))
        )
        if self.opts.get('minion_data_cache', False):
            ref = {'G': self._check_grain_minions,
                   'P': self._check_grain_pcre_minions,
                   'I': self._check_pillar_minions,
                   'L': self._check_list_minions,
                   'S': self._check_ipcidr_minions,
                   'E': self._check_pcre_minions,
                   'R': self._all_minions}
            results = []
            unmatched = []
            opers = ['and', 'or', 'not', '(', ')']
            tokens = expr.split()
            for match in tokens:
                # Try to match tokens from the compound target, first by using
                # the 'G, X, I, L, S, E' matcher types, then by hostname glob.
                if '@' in match and match[1] == '@':
                    comps = match.split('@')
                    matcher = ref.get(comps[0])
                    if not matcher:
                        # If an unknown matcher is called at any time, fail out
                        return []
                    if unmatched and unmatched[-1] == '-':
                        results.append(str(set(matcher('@'.join(comps[1:])))))
                        results.append(')')
                        unmatched.pop()
                    else:
                        results.append(str(set(matcher('@'.join(comps[1:])))))
                elif match in opers:
                    # We didn't match a target, so append a boolean operator or
                    # subexpression
                    if results:
                        if match == 'not':
                            if results[-1] == '&':
                                pass
                            elif results[-1] == '|':
                                pass
                            else:
                                results.append('&')
                            results.append('(')
                            results.append(str(set(minions)))
                            results.append('-')
                            unmatched.append('-')
                        elif match == 'and':
                            results.append('&')
                        elif match == 'or':
                            results.append('|')
                        elif match == '(':
                            results.append(match)
                            unmatched.append(match)
                        elif match == ')':
                            if not unmatched or unmatched[-1] != '(':
                                log.error('Invalid compound expr (unexpected '
                                          'right parenthesis): {0}'
                                          .format(expr))
                                return []
                            results.append(match)
                            unmatched.pop()
                            if unmatched and unmatched[-1] == '-':
                                results.append(')')
                                unmatched.pop()
                        else:  # Won't get here, unless oper is added
                            log.error('Unhandled oper in compound expr: {0}'
                                      .format(expr))
                            return []
                    else:
                        # seq start with oper, fail
                        if match == '(':
                            results.append(match)
                            unmatched.append(match)
                        else:
                            return []
                else:
                    # The match is not explicitly defined, evaluate as a glob
                    if unmatched and unmatched[-1] == '-':
                        results.append(
                                str(set(self._check_glob_minions(match))))
                        results.append(')')
                        unmatched.pop()
                    else:
                        results.append(
                                str(set(self._check_glob_minions(match))))
            for token in unmatched:
                results.append(')')
            results = ' '.join(results)
            try:
                return list(eval(results))
            except Exception:
                log.error('Invalid compound target: {0}'.format(expr))
                return []
        return list(minions)

    def _all_minions(self, expr=None):
        '''
        Return a list of all minions that have auth'd<commit_msg>Add debug log of final set logic for compound match<commit_after>'''
This module contains routines used to verify the matcher against the minions
expected to return
'''

# Import python libs
import os
import glob
import re
import logging

# Import salt libs
import salt.payload
import salt.utils

log = logging.getLogger(__name__)


def nodegroup_comp(group, nodegroups, skip=None):
    '''
    Take the nodegroup and the nodegroups and fill in nodegroup refs
    '''
    if skip is None:
        skip = set([group])
    if group not in nodegroups:
        return ''
    gstr = nodegroups[group]
    ret = ''
    for comp in gstr.split():
        if not comp.startswith('N@'):
            ret += '{0} '.format(comp)
            continue
        ngroup = comp[2:]
        if ngroup in skip:
            continue
        skip.add(ngroup)
        ret += nodegroup_comp(ngroup, nodegroups, skip)
    return ret


class CkMinions(object):
    '''
    Used to check what minions should respond from a target
    '''
    def __init__(self, opts):
        self.opts = opts
        self.serial = salt.payload.Serial(opts)

    def _check_glob_minions(self, expr):
        '''
        Return the minions found by looking via globs
        '''
        cwd = os.getcwd()
        os.chdir(os.path.join(self.opts['pki_dir'], 'minions'))
        ret = set(glob.glob(expr))
        os.chdir(cwd)
        return list(ret)

    def _check_list_minions(self, expr):
        '''
        Return the minions found by looking via a list
        '''
        ret = []
        for fn_ in os.listdir(os.path.join(self.opts['pki_dir'], 'minions')):
            if fn_ in expr:
                if fn_ not in ret:
                    ret.append(fn_)
        return ret

    def _check_pcre_minions(self, expr):
        '''
        Return the minions found by looking via regular expressions
        '''
        cwd = os.getcwd()
        os.chdir(os.path.join(self.opts['pki_dir'], 'minions'))
        reg = re.compile(expr)
        ret = [fn_ for fn_ in os.listdir('.') if reg.match(fn_)]
        os.chdir(cwd)
        return ret

    def _check_grain_minions(self, expr):
        '''
        Return the minions found by looking via grains
        '''
        minions = set(
            os.listdir(os.path.join(self.opts['pki_dir'], 'minions'))
        )
        if self.opts.get('minion_data_cache', False):
            cdir = os.path.join(self.opts['cachedir'], 'minions')
            if not os.path.isdir(cdir):
                return list(minions)
            for id_ in os.listdir(cdir):
                if id_ not in minions:
                    continue
                datap = os.path.join(cdir, id_, 'data.p')
                if not os.path.isfile(datap):
                    continue
                grains = self.serial.load(
                    salt.utils.fopen(datap)
                ).get('grains')
                if not salt.utils.subdict_match(grains, expr):
                    minions.remove(id_)
        return list(minions)

    def _check_grain_pcre_minions(self, expr):
        '''
        Return the minions found by looking via grains with PCRE
        '''
        minions = set(
            os.listdir(os.path.join(self.opts['pki_dir'], 'minions'))
        )
        if self.opts.get('minion_data_cache', False):
            cdir = os.path.join(self.opts['cachedir'], 'minions')
            if not os.path.isdir(cdir):
                return list(minions)
            for id_ in os.listdir(cdir):
                if id_ not in minions:
                    continue
                datap = os.path.join(cdir, id_, 'data.p')
                if not os.path.isfile(datap):
                    continue
                grains = self.serial.load(
                    salt.utils.fopen(datap)
                ).get('grains')
                if not salt.utils.subdict_match(grains, expr,
                                                delim=':', regex_match=True):
                    minions.remove(id_)
        return list(minions)

    def _check_pillar_minions(self, expr):
        '''
        Return the minions found by looking via pillar
        '''
        minions = set(
            os.listdir(os.path.join(self.opts['pki_dir'], 'minions'))
        )
        if self.opts.get('minion_data_cache', False):
            cdir = os.path.join(self.opts['cachedir'], 'minions')
            if not os.path.isdir(cdir):
                return list(minions)
            for id_ in os.listdir(cdir):
                if id_ not in minions:
                    continue
                datap = os.path.join(cdir, id_, 'data.p')
                if not os.path.isfile(datap):
                    continue
                pillar = self.serial.load(
                    salt.utils.fopen(datap)
                ).get('pillar')
                if not salt.utils.subdict_match(pillar, expr):
                    minions.remove(id_)
        return list(minions)

    def _check_ipcidr_minions(self, expr):
        '''
        Return the minions found by looking via ipcidr
        '''
        minions = set(
            os.listdir(os.path.join(self.opts['pki_dir'], 'minions'))
        )
        if self.opts.get('minion_data_cache', False):
            cdir = os.path.join(self.opts['cachedir'], 'minions')
            if not os.path.isdir(cdir):
                return list(minions)
            for id_ in os.listdir(cdir):
                if id_ not in minions:
                    continue
                datap = os.path.join(cdir, id_, 'data.p')
                if not os.path.isfile(datap):
                    continue
                grains = self.serial.load(
                    salt.utils.fopen(datap)
                ).get('grains')

                num_parts = len(expr.split('/'))
                if num_parts > 2:
                    # Target is not valid CIDR, no minions match
                    return []
                elif num_parts == 2:
                    # Target is CIDR
                    if not salt.utils.network.in_subnet(
                            expr,
                            addrs=grains.get('ipv4', [])):
                        minions.remove(id_)
                else:
                    # Target is an IPv4 address
                    import socket
                    try:
                        socket.inet_aton(expr)
                    except socket.error:
                        # Not a valid IPv4 address, no minions match
                        return []
                    else:
                        if not expr in grains.get('ipv4', []):
                            minions.remove(id_)
        return list(minions)

    def _check_compound_minions(self, expr):
        '''
        Return the minions found by looking via compound matcher
        '''
        minions = set(
            os.listdir(os.path.join(self.opts['pki_dir'], 'minions'))
        )
        if self.opts.get('minion_data_cache', False):
            ref = {'G': self._check_grain_minions,
                   'P': self._check_grain_pcre_minions,
                   'I': self._check_pillar_minions,
                   'L': self._check_list_minions,
                   'S': self._check_ipcidr_minions,
                   'E': self._check_pcre_minions,
                   'R': self._all_minions}
            results = []
            unmatched = []
            opers = ['and', 'or', 'not', '(', ')']
            tokens = expr.split()
            for match in tokens:
                # Try to match tokens from the compound target, first by using
                # the 'G, X, I, L, S, E' matcher types, then by hostname glob.
                if '@' in match and match[1] == '@':
                    comps = match.split('@')
                    matcher = ref.get(comps[0])
                    if not matcher:
                        # If an unknown matcher is called at any time, fail out
                        return []
                    if unmatched and unmatched[-1] == '-':
                        results.append(str(set(matcher('@'.join(comps[1:])))))
                        results.append(')')
                        unmatched.pop()
                    else:
                        results.append(str(set(matcher('@'.join(comps[1:])))))
                elif match in opers:
                    # We didn't match a target, so append a boolean operator or
                    # subexpression
                    if results:
                        if match == 'not':
                            if results[-1] == '&':
                                pass
                            elif results[-1] == '|':
                                pass
                            else:
                                results.append('&')
                            results.append('(')
                            results.append(str(set(minions)))
                            results.append('-')
                            unmatched.append('-')
                        elif match == 'and':
                            results.append('&')
                        elif match == 'or':
                            results.append('|')
                        elif match == '(':
                            results.append(match)
                            unmatched.append(match)
                        elif match == ')':
                            if not unmatched or unmatched[-1] != '(':
                                log.error('Invalid compound expr (unexpected '
                                          'right parenthesis): {0}'
                                          .format(expr))
                                return []
                            results.append(match)
                            unmatched.pop()
                            if unmatched and unmatched[-1] == '-':
                                results.append(')')
                                unmatched.pop()
                        else:  # Won't get here, unless oper is added
                            log.error('Unhandled oper in compound expr: {0}'
                                      .format(expr))
                            return []
                    else:
                        # seq start with oper, fail
                        if match == '(':
                            results.append(match)
                            unmatched.append(match)
                        else:
                            return []
                else:
                    # The match is not explicitly defined, evaluate as a glob
                    if unmatched and unmatched[-1] == '-':
                        results.append(
                                str(set(self._check_glob_minions(match))))
                        results.append(')')
                        unmatched.pop()
                    else:
                        results.append(
                                str(set(self._check_glob_minions(match))))
            for token in unmatched:
                results.append(')')
            results = ' '.join(results)
            log.debug('Evaluating final compound matching expr: {0}'
                      .format(results))
            try:
                return list(eval(results))
            except Exception:
                log.error('Invalid compound target: {0}'.format(expr))
                return []
        return list(minions)

    def _all_minions(self, expr=None):
        '''
        Return a list of all minions that have auth'd
{'fraction_non_alphanumeric': 0.10191725529767912, 'fraction_numerical': 0.0018865441144210941, 'mean_word_length': 1.1550534177933252, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '881481', 'n_tokens_mistral': 6095, 'n_tokens_neox': 5808, 'n_words': 1445}