<commit_before>/*
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

/*
 * Copyright 2020 Joyent, Inc.
 */

var fs = require('fs');

var artedi = require('artedi');
var assert = require('assert-plus');
var bunyan = require('bunyan');
var LRU = require('lru-cache');
var restify = require('restify');

var uploadsCommon = require('./uploads/common');

///--- Constants

const DEF_MAX_STREAMING_SIZE_MB = 51200;
const DEF_MAX_PERCENT_UTIL = 90;
const DEF_MAX_OPERATOR_PERCENT_UTIL = 92;

/**
 * Configure the application based on the configuration file data and the
 * command line options.
 *
 * @param {String} appName: Required. The name of the application.
 * @param {Object} opts: Required. An object representing the parsed command
 * line options.
 * @param {} dtProbes: Required. An object containing the dtrace probes for the
 * application.
 * @returns {Object} The configuration object.
 */
function configure(appName, opts, dtProbes) {
    var cfg = JSON.parse(readFile(opts.file));

    assert.object(cfg, 'cfg');
    assert.object(cfg.auth, 'cfg.auth');
    assert.object(cfg.moray, 'cfg.moray');
    assert.object(cfg.cueballHttpAgent, 'cfg.cueballHttpAgent');
    assert.object(cfg.sharkConfig, 'cfg.sharkConfig');
    assert.optionalObject(cfg.storage, 'cfg.storage');

    /* Used by artedi and RBAC */
    assert.string(cfg.datacenter, 'cfg.datacenter');
    assert.string(cfg.region, 'cfg.region');

    cfg.insecurePort = opts.insecure_port || cfg.insecurePort;
    cfg.port = opts.port || cfg.port;
    cfg.name = 'Manta/2';
    cfg.version = apiVersion();
    cfg.log = configureLogging(appName, cfg.bunyan, opts.verbose);

    [ cfg.auth, cfg.moray,
      cfg.cueballHttpAgent, cfg.sharkConfig
    ].forEach(function (x) {
        x.log = cfg.log;
    });

    if (!cfg.hasOwnProperty('storage')) {
        cfg.storage = {};
    }

    /*
     * For streaming PUTs and multi-part uploads, we may not know the full size
     * of the object until the completion of the request.  For object
     * placement, we must know in advance the maximum expected size of the
     * stream.  If the client does not provide a "Max-Content-Length" header,
     * we assume a default value.  An operator may override this value by using
     * the "MUSKIE_DEFAULT_MAX_STREAMING_SIZE_MB" SAPI property.
     */
    setNumericConfigProperty('defaultMaxStreamingSizeMB',
        DEF_MAX_STREAMING_SIZE_MB, cfg.storage, cfg.log,
        function (x) { return (x >= 1); });

    if (!cfg.hasOwnProperty('accountsSnaplinksDisabled')) {
        cfg.accountsSnaplinksDisabled = [];
    } else {
        assert.arrayOfObject(cfg.accountsSnaplinksDisabled,
            'cfg.accountsSnaplinksDisabled');
        for (var i = 0; i < cfg.accountsSnaplinksDisabled.length; i++) {
            var uuid = cfg.accountsSnaplinksDisabled[i].uuid;

            assert.uuid(uuid, 'cfg.accountsSnaplinksDisabled[i].uuid');
            cfg.log.info('snaplinks disabled for uuid ' + uuid);
        }
    }

    if (!cfg.hasOwnProperty('multipartUpload')) {
        cfg.multipartUpload = {};
    }

    if (cfg.multipartUpload.hasOwnProperty('prefixDirLen')) {
        var len = cfg.multipartUpload.prefixDirLen;
        assert.number(len, '"prefixDirLen" value must be a number');

        if (len < uploadsCommon.MIN_PREFIX_LEN ||
            len > uploadsCommon.MAX_PREFIX_LEN) {

            cfg.log.fatal('invalid "prefixDirLen" value: must be between ' +
                uploadsCommon.MIN_PREFIX_LEN + ' and ' +
                uploadsCommon.MAX_PREFIX_LEN);
            process.exit(1);
        }
    } else {
        cfg.multipartUpload.prefixDirLen = uploadsCommon.DEF_PREFIX_LEN;
    }

    setNumericConfigProperty('maxUtilizationPct',
        DEF_MAX_PERCENT_UTIL, cfg.storage, cfg.log,
        function (x) { return (x > 0 && x <= 100); });
    setNumericConfigProperty('maxOperatorUtilizationPct',
        DEF_MAX_OPERATOR_PERCENT_UTIL, cfg.storage, cfg.log,
        function (x) { return (x > 0 && x <= 100); });

    /*
     * If the configuration is invalid such that the maximum utilization for
     * normal request is greater than the maximum utilization for operator
     * requests then log a warning and use the greater of the default maximum
     * operator utilization or the maximum utilization for normal requests.
     */
    if (cfg.storage.maxUtilizationPct > cfg.storage.maxOperatorUtilizationPct) {
        if (DEF_MAX_OPERATOR_PERCENT_UTIL > cfg.storage.maxUtilizationPct) {
            cfg.log.warn('invalid configuration: "maxUtilizationPct" value (' +
                cfg.storage.maxUtilizationPct + ') should not exceed the ' +
                'value for maxOperatorUtilizationPct (' +
                cfg.storage.maxOperatorUtilizationPct + '). Using the default' +
                ' operator utilization value of ' +
                DEF_MAX_OPERATOR_PERCENT_UTIL + ' as the value for ' +
                'maxOperatorUtilizationPct.');
            cfg.storage.maxOperatorUtilizationPct =
                DEF_MAX_OPERATOR_PERCENT_UTIL;
        } else {
            cfg.log.warn('invalid configuration: "maxUtilizationPct" value (' +
                cfg.storage.maxUtilizationPct + ') should not exceed the ' +
                'value for maxOperatorUtilizationPct (' +
                cfg.storage.maxOperatorUtilizationPct + '). Using ' +
                cfg.storage.maxUtilizationPct + ' as the value for ' +
                'maxOperatorUtilizationPct.');
            cfg.storage.maxOperatorUtilizationPct =
                cfg.storage.maxUtilizationPct;
        }
    }

    cfg.collector = artedi.createCollector({
        labels: {
            datacenter: cfg.datacenter,
            server: cfg.server_uuid,
            zonename: cfg.zone_uuid,
            pid: process.pid
        }
    });

    cfg.dtrace_probes = dtProbes;

    cfg.log.debug(cfg, 'muskie: config loaded');

    return (cfg);
}


/**
 * Configure the logger based on the configuration data
 *
 * @param {String} appName: Required. The name of the application
 * @param {Object} bunyanCfg: Optional. The bunyan configuration data
 * @param {arrayOfBool} verbose: Optional. Array of boolean values that
 * indicates if verbose logging should be enabled and the level of verbosity
 * requested.
 * @returns {Object} A bunyan logger object
 */
function configureLogging(appName, bunyanCfg, verbose) {
    assert.optionalObject(bunyanCfg, 'config.bunyan');

    var level;
    if (bunyanCfg) {
        assert.optionalString(bunyanCfg.level, 'config.bunyan.level');

        level = bunyan.resolveLevel(process.env.LOG_LEVEL ||
                    bunyanCfg.level || 'info');
    } else {
        level = bunyan.resolveLevel(process.env.LOG_LEVEL || 'info');
    }
<commit_msg>MANTA-5352 muskie 'muskie: config loaded' log record is accidentally huge (#77)<commit_after>/*
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

/*
 * Copyright 2020 Joyent, Inc.
 */

var fs = require('fs');

var artedi = require('artedi');
var assert = require('assert-plus');
var bunyan = require('bunyan');
var LRU = require('lru-cache');
var restify = require('restify');

var uploadsCommon = require('./uploads/common');
var utils = require('./utils');

///--- Constants

const DEF_MAX_STREAMING_SIZE_MB = 51200;
const DEF_MAX_PERCENT_UTIL = 90;
const DEF_MAX_OPERATOR_PERCENT_UTIL = 92;

/**
 * Configure the application based on the configuration file data and the
 * command line options.
 *
 * @param {String} appName: Required. The name of the application.
 * @param {Object} opts: Required. An object representing the parsed command
 * line options.
 * @param {} dtProbes: Required. An object containing the dtrace probes for the
 * application.
 * @returns {Object} The configuration object.
 */
function configure(appName, opts, dtProbes) {
    var cfg = JSON.parse(readFile(opts.file));

    assert.object(cfg, 'cfg');
    assert.object(cfg.auth, 'cfg.auth');
    assert.object(cfg.moray, 'cfg.moray');
    assert.object(cfg.cueballHttpAgent, 'cfg.cueballHttpAgent');
    assert.object(cfg.sharkConfig, 'cfg.sharkConfig');
    assert.optionalObject(cfg.storage, 'cfg.storage');

    /* Used by artedi and RBAC */
    assert.string(cfg.datacenter, 'cfg.datacenter');
    assert.string(cfg.region, 'cfg.region');

    var log = configureLogging(appName, cfg.bunyan, opts.verbose);

    cfg.insecurePort = opts.insecure_port || cfg.insecurePort;
    cfg.port = opts.port || cfg.port;
    cfg.name = 'Manta/2';
    cfg.version = apiVersion();

    if (log.debug()) {
        // Log a "reasonable" set of the app config.
        // - Elide password/key stuff.
        // - Avoid logging `cfg.log` and similar which results in an accidental
        //   huge log record.
        var loggableCfg = utils.shallowCopy(cfg);
        delete loggableCfg.authToken;
        log.debug({loggableCfg: loggableCfg}, 'loggable config');
    }

    cfg.log = log;
    [ cfg.auth, cfg.moray,
      cfg.cueballHttpAgent, cfg.sharkConfig
    ].forEach(function (x) {
        x.log = cfg.log;
    });

    if (!cfg.hasOwnProperty('storage')) {
        cfg.storage = {};
    }

    /*
     * For streaming PUTs and multi-part uploads, we may not know the full size
     * of the object until the completion of the request.  For object
     * placement, we must know in advance the maximum expected size of the
     * stream.  If the client does not provide a "Max-Content-Length" header,
     * we assume a default value.  An operator may override this value by using
     * the "MUSKIE_DEFAULT_MAX_STREAMING_SIZE_MB" SAPI property.
     */
    setNumericConfigProperty('defaultMaxStreamingSizeMB',
        DEF_MAX_STREAMING_SIZE_MB, cfg.storage, cfg.log,
        function (x) { return (x >= 1); });

    if (!cfg.hasOwnProperty('accountsSnaplinksDisabled')) {
        cfg.accountsSnaplinksDisabled = [];
    } else {
        assert.arrayOfObject(cfg.accountsSnaplinksDisabled,
            'cfg.accountsSnaplinksDisabled');
        for (var i = 0; i < cfg.accountsSnaplinksDisabled.length; i++) {
            var uuid = cfg.accountsSnaplinksDisabled[i].uuid;

            assert.uuid(uuid, 'cfg.accountsSnaplinksDisabled[i].uuid');
            cfg.log.info('snaplinks disabled for uuid ' + uuid);
        }
    }

    if (!cfg.hasOwnProperty('multipartUpload')) {
        cfg.multipartUpload = {};
    }

    if (cfg.multipartUpload.hasOwnProperty('prefixDirLen')) {
        var len = cfg.multipartUpload.prefixDirLen;
        assert.number(len, '"prefixDirLen" value must be a number');

        if (len < uploadsCommon.MIN_PREFIX_LEN ||
            len > uploadsCommon.MAX_PREFIX_LEN) {

            cfg.log.fatal('invalid "prefixDirLen" value: must be between ' +
                uploadsCommon.MIN_PREFIX_LEN + ' and ' +
                uploadsCommon.MAX_PREFIX_LEN);
            process.exit(1);
        }
    } else {
        cfg.multipartUpload.prefixDirLen = uploadsCommon.DEF_PREFIX_LEN;
    }

    setNumericConfigProperty('maxUtilizationPct',
        DEF_MAX_PERCENT_UTIL, cfg.storage, cfg.log,
        function (x) { return (x > 0 && x <= 100); });
    setNumericConfigProperty('maxOperatorUtilizationPct',
        DEF_MAX_OPERATOR_PERCENT_UTIL, cfg.storage, cfg.log,
        function (x) { return (x > 0 && x <= 100); });

    /*
     * If the configuration is invalid such that the maximum utilization for
     * normal request is greater than the maximum utilization for operator
     * requests then log a warning and use the greater of the default maximum
     * operator utilization or the maximum utilization for normal requests.
     */
    if (cfg.storage.maxUtilizationPct > cfg.storage.maxOperatorUtilizationPct) {
        if (DEF_MAX_OPERATOR_PERCENT_UTIL > cfg.storage.maxUtilizationPct) {
            cfg.log.warn('invalid configuration: "maxUtilizationPct" value (' +
                cfg.storage.maxUtilizationPct + ') should not exceed the ' +
                'value for maxOperatorUtilizationPct (' +
                cfg.storage.maxOperatorUtilizationPct + '). Using the default' +
                ' operator utilization value of ' +
                DEF_MAX_OPERATOR_PERCENT_UTIL + ' as the value for ' +
                'maxOperatorUtilizationPct.');
            cfg.storage.maxOperatorUtilizationPct =
                DEF_MAX_OPERATOR_PERCENT_UTIL;
        } else {
            cfg.log.warn('invalid configuration: "maxUtilizationPct" value (' +
                cfg.storage.maxUtilizationPct + ') should not exceed the ' +
                'value for maxOperatorUtilizationPct (' +
                cfg.storage.maxOperatorUtilizationPct + '). Using ' +
                cfg.storage.maxUtilizationPct + ' as the value for ' +
                'maxOperatorUtilizationPct.');
            cfg.storage.maxOperatorUtilizationPct =
                cfg.storage.maxUtilizationPct;
        }
    }

    cfg.collector = artedi.createCollector({
        labels: {
            datacenter: cfg.datacenter,
            server: cfg.server_uuid,
            zonename: cfg.zone_uuid,
            pid: process.pid
        }
    });

    cfg.dtrace_probes = dtProbes;

    log.debug('config loaded');

    return (cfg);
}


/**
 * Configure the logger based on the configuration data
 *
 * @param {String} appName: Required. The name of the application
 * @param {Object} bunyanCfg: Optional. The bunyan configuration data
 * @param {arrayOfBool} verbose: Optional. Array of boolean values that
 * indicates if verbose logging should be enabled and the level of verbosity
 * requested.
 * @returns {Object} A bunyan logger object
 */
function configureLogging(appName, bunyanCfg, verbose) {
    assert.optionalObject(bunyanCfg, 'config.bunyan');

    var level;
    if (bunyanCfg) {
        assert.optionalString(bunyanCfg.level, 'config.bunyan.level');

        level = bunyan.resolveLevel(process.env.LOG_LEVEL ||
                    bunyanCfg.level || 'info');
    } else {
        level = bunyan.resolveLevel(process.env.LOG_LEVEL || 'info');
    }

{'fraction_non_alphanumeric': 0.1095135059677532, 'fraction_numerical': 0.004467090109583304, 'mean_word_length': 2.7390396659707723, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7272491', 'n_tokens_mistral': 4347, 'n_tokens_neox': 4199, 'n_words': 1083}