<commit_before>import os.path
import logging
import json
import tempfile
import shutil
import time

from ceph_deploy import hosts
from ceph_deploy.cliutil import priority
from ceph_deploy.lib import remoto
import ceph_deploy.util.paths.mon

LOG = logging.getLogger(__name__)


def _keyring_equivalent(keyring_one, keyring_two):
    """
    Check two keyrings are identical
    """
    def keyring_extract_key(file_path):
        """
        Cephx keyring files may or may not have white space before some lines.
        They may have some values in quotes, so a safe way to compare is to
        extract the key.
        """
        with file(file_path, 'r') as f:
            for line in f:
                content = line.strip()
                if len(content) == 0:
                    continue
                split_line = content.split('=')
                if split_line[0].strip() == 'key':
                    return "=".join(split_line[1:]).strip()
        raise RuntimeError("File '%s' is not a keyring" % file_path)
    key_one = keyring_extract_key(keyring_one)
    key_two = keyring_extract_key(keyring_two)
    return key_one == key_two


def keytype_path_to(args, keytype):
    """
    Get the local filename for a keyring type
    """
    if keytype == "admin":
        return '{cluster}.client.admin.keyring'.format(
            cluster=args.cluster)
    if keytype == "mon":
        return '{cluster}.mon.keyring'.format(
            cluster=args.cluster)
    return '{cluster}.bootstrap-{what}.keyring'.format(
            cluster=args.cluster,
            what=keytype)


def keytype_identity(keytype):
    """
    Get the keyring identity from keyring type.

    This is used in authentication with keyrings and generating keyrings.
    """
    ident_dict = {
        'admin' : 'client.admin',
        'mds' : 'client.bootstrap-mds',
        'osd' : 'client.bootstrap-osd',
        'rgw' : 'client.bootstrap-rgw',
        'mon' : 'mon.'
    }
    return ident_dict.get(keytype, None)


def keytype_capabilities(keytype):
    """
    Get the capabilities of a keyring from keyring type.
    """
    cap_dict = {
        'admin' : [
            'osd', 'allow *',
            'mds', 'allow *',
            'mon', 'allow *'
            ],
        'mds' : [
            'mon', 'allow profile bootstrap-mds'
            ],
        'osd' : [
            'mon', 'allow profile bootstrap-osd'
            ],
        'rgw': [
            'mon', 'allow profile bootstrap-rgw'
            ]
        }
    return cap_dict.get(keytype, None)


def gatherkeys_missing(args, distro, rlogger, keypath, keytype, dest_dir):
    """
    Get or create the keyring from the mon using the mon keyring by keytype and
    copy to dest_dir
    """
    arguments = [
        '/usr/bin/ceph',
        '--connect-timeout=25',
        '--cluster={cluster}'.format(
            cluster=args.cluster),
        '--name', 'mon.',
        '--keyring={keypath}'.format(
            keypath=keypath),
        'auth', 'get-or-create',
        ]
    identity = keytype_identity(keytype)
    if identity is None:
        raise RuntimeError('Could not find identity for keytype:%s' % keytype)
    arguments.append(identity)
    capabilites = keytype_capabilities(keytype)
    if capabilites is None:
        raise RuntimeError('Could not find capabilites for keytype:%s' % keytype)
    arguments.extend(capabilites)
    out, err, code = remoto.process.check(
        distro.conn,
        arguments
        )
    if code != 0:
        rlogger.error('"ceph auth get-or-create for keytype %s returned %s', keytype, code)
        for line in err:
            rlogger.debug(line)
        return False
    keyring_name_local = keytype_path_to(args, keytype)
    keyring_path_local = os.path.join(dest_dir, keyring_name_local)
    with file(keyring_path_local, 'w') as f:
        for line in out:
            f.write(line + '\n')
    return True


def gatherkeys_with_mon(args, host, dest_dir):
    """
    Connect to mon and gather keys if mon is in quorum.
    """
    distro = hosts.get(host, username=args.username)
    dir_keytype_mon = ceph_deploy.util.paths.mon.path(args.cluster, host)
    path_keytype_mon = "%s/keyring" % (dir_keytype_mon)
    mon_key = distro.conn.remote_module.get_file(path_keytype_mon)
    if mon_key is None:
        LOG.warning("No mon key found in host: %s", host)
        return False
    mon_name_local = keytype_path_to(args, "mon")
    mon_path_local = os.path.join(dest_dir, mon_name_local)
    with file(mon_path_local, 'w') as f:
        f.write(mon_key)
    rlogger = logging.getLogger(host)
    path_asok = ceph_deploy.util.paths.mon.asok(args.cluster, host)
    out, err, code = remoto.process.check(
        distro.conn,
            [
                "/usr/bin/ceph",
                "--connect-timeout=25",
                "--cluster={cluster}".format(
                    cluster=args.cluster),
                "--admin-daemon={asok}".format(
                    asok=path_asok),
                "mon_status"
            ]
        )
    if code != 0:
        rlogger.error('"ceph mon_status %s" returned %s', host, code)
        for line in err:
            rlogger.debug(line)
        return False
    try:
        mon_status = json.loads("".join(out))
    except ValueError:<commit_msg>[fix-remove-ceph-create-keys-dep] ceph_deploy.gatherkeys: logging mon keyring location<commit_after>import os.path
import logging
import json
import tempfile
import shutil
import time

from ceph_deploy import hosts
from ceph_deploy.cliutil import priority
from ceph_deploy.lib import remoto
import ceph_deploy.util.paths.mon

LOG = logging.getLogger(__name__)


def _keyring_equivalent(keyring_one, keyring_two):
    """
    Check two keyrings are identical
    """
    def keyring_extract_key(file_path):
        """
        Cephx keyring files may or may not have white space before some lines.
        They may have some values in quotes, so a safe way to compare is to
        extract the key.
        """
        with file(file_path, 'r') as f:
            for line in f:
                content = line.strip()
                if len(content) == 0:
                    continue
                split_line = content.split('=')
                if split_line[0].strip() == 'key':
                    return "=".join(split_line[1:]).strip()
        raise RuntimeError("File '%s' is not a keyring" % file_path)
    key_one = keyring_extract_key(keyring_one)
    key_two = keyring_extract_key(keyring_two)
    return key_one == key_two


def keytype_path_to(args, keytype):
    """
    Get the local filename for a keyring type
    """
    if keytype == "admin":
        return '{cluster}.client.admin.keyring'.format(
            cluster=args.cluster)
    if keytype == "mon":
        return '{cluster}.mon.keyring'.format(
            cluster=args.cluster)
    return '{cluster}.bootstrap-{what}.keyring'.format(
            cluster=args.cluster,
            what=keytype)


def keytype_identity(keytype):
    """
    Get the keyring identity from keyring type.

    This is used in authentication with keyrings and generating keyrings.
    """
    ident_dict = {
        'admin' : 'client.admin',
        'mds' : 'client.bootstrap-mds',
        'osd' : 'client.bootstrap-osd',
        'rgw' : 'client.bootstrap-rgw',
        'mon' : 'mon.'
    }
    return ident_dict.get(keytype, None)


def keytype_capabilities(keytype):
    """
    Get the capabilities of a keyring from keyring type.
    """
    cap_dict = {
        'admin' : [
            'osd', 'allow *',
            'mds', 'allow *',
            'mon', 'allow *'
            ],
        'mds' : [
            'mon', 'allow profile bootstrap-mds'
            ],
        'osd' : [
            'mon', 'allow profile bootstrap-osd'
            ],
        'rgw': [
            'mon', 'allow profile bootstrap-rgw'
            ]
        }
    return cap_dict.get(keytype, None)


def gatherkeys_missing(args, distro, rlogger, keypath, keytype, dest_dir):
    """
    Get or create the keyring from the mon using the mon keyring by keytype and
    copy to dest_dir
    """
    arguments = [
        '/usr/bin/ceph',
        '--connect-timeout=25',
        '--cluster={cluster}'.format(
            cluster=args.cluster),
        '--name', 'mon.',
        '--keyring={keypath}'.format(
            keypath=keypath),
        'auth', 'get-or-create',
        ]
    identity = keytype_identity(keytype)
    if identity is None:
        raise RuntimeError('Could not find identity for keytype:%s' % keytype)
    arguments.append(identity)
    capabilites = keytype_capabilities(keytype)
    if capabilites is None:
        raise RuntimeError('Could not find capabilites for keytype:%s' % keytype)
    arguments.extend(capabilites)
    out, err, code = remoto.process.check(
        distro.conn,
        arguments
        )
    if code != 0:
        rlogger.error('"ceph auth get-or-create for keytype %s returned %s', keytype, code)
        for line in err:
            rlogger.debug(line)
        return False
    keyring_name_local = keytype_path_to(args, keytype)
    keyring_path_local = os.path.join(dest_dir, keyring_name_local)
    with file(keyring_path_local, 'w') as f:
        for line in out:
            f.write(line + '\n')
    return True


def gatherkeys_with_mon(args, host, dest_dir):
    """
    Connect to mon and gather keys if mon is in quorum.
    """
    distro = hosts.get(host, username=args.username)
    dir_keytype_mon = ceph_deploy.util.paths.mon.path(args.cluster, host)
    path_keytype_mon = "%s/keyring" % (dir_keytype_mon)
    mon_key = distro.conn.remote_module.get_file(path_keytype_mon)
    if mon_key is None:
        LOG.warning("No mon key not found %s:%s", host, path_keytype_mon)
        return False
    LOG.info("Found keyring at %s:%s", host, path_keytype_mon)
    mon_name_local = keytype_path_to(args, "mon")
    mon_path_local = os.path.join(dest_dir, mon_name_local)
    with file(mon_path_local, 'w') as f:
        f.write(mon_key)
    rlogger = logging.getLogger(host)
    path_asok = ceph_deploy.util.paths.mon.asok(args.cluster, host)
    out, err, code = remoto.process.check(
        distro.conn,
            [
                "/usr/bin/ceph",
                "--connect-timeout=25",
                "--cluster={cluster}".format(
                    cluster=args.cluster),
                "--admin-daemon={asok}".format(
                    asok=path_asok),
                "mon_status"
            ]
        )
    if code != 0:
        rlogger.error('"ceph mon_status %s" returned %s', host, code)
        for line in err:
            rlogger.debug(line)
        return False
    try:
        mon_status = json.loads("".join(out))
    except ValueError:
{'fraction_non_alphanumeric': 0.11452721530744937, 'fraction_numerical': 0.0016719301504737136, 'mean_word_length': 2.1621145374449338, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4088308', 'n_tokens_mistral': 3395, 'n_tokens_neox': 3309, 'n_words': 820}