<commit_before>/*
 * Copyright 2015 Cloudius Systems
 *
 */

#pragma once

#include <boost/program_options.hpp>
#include <unordered_map>
#include "core/sstring.hh"
#include "core/future.hh"

class file;

namespace db {

typedef std::unordered_map<sstring, sstring> string_map;

/*
 * This type is not use, and probably never will be.
 * So it makes sense to jump through hoops just to ensure
 * it is in fact handled properly...
 */
struct seed_provider_type {
    seed_provider_type() = default;
    seed_provider_type(sstring n,
            std::initializer_list<string_map::value_type> opts =
                    { })
            : class_name(std::move(n)), parameters(std::move(opts)) {
    }
    sstring class_name;
    string_map parameters;
};

class config {
public:
    enum class value_status {
        Used,
        Unused,
        Invalid,
    };

    enum class config_source : uint8_t {
        None,
        SettingsFile,
        CommandLine
    };

    template<typename T, value_status S>
    struct value {
        typedef T type;
        typedef value<T, S> MyType;

        value(const T& t = T()) : _value(t)
        {}
        value_status status() const {
            return S;
        }
        config_source source() const {
            return _source;
        }
        bool is_set() const {
            return _source > config_source::None;
        }
        MyType & operator()(const T& t) {
            _value = t;
            return *this;
        }
        MyType & operator()(T&& t) {
            _value = std::move(t);
            return *this;
        }
        const T& operator()() const {
            return _value;
        }
        T& operator()() {
            return _value;
        }
    private:
        friend class config;
        T _value = T();
        config_source _source = config_source::None;
    };

    config();

    boost::program_options::options_description
    get_options_description();

    boost::program_options::options_description_easy_init&
    add_options(boost::program_options::options_description_easy_init&);

    void read_from_yaml(const sstring&);
    void read_from_yaml(const char *);
    future<> read_from_file(const sstring&);
    future<> read_from_file(file);

    typedef std::unordered_map<sstring, sstring> string_map;
    typedef std::vector<sstring> string_list;
    using seed_provider_type = db::seed_provider_type;

    /*
     * All values and documentation taken from
     * http://docs.datastax.com/en/cassandra/2.1/cassandra/configuration/configCassandra_yaml_r.html
     *
     * Big fat x-macro expansion of _all_ (at this writing) cassandra opts, with the "documentation"
     * included.
     *
     * X-macro syntax:
     *
     * X(member, type, default, status, desc [, value, value...])
     *
     * Where
     *  member: is the property name -> config member name
     *  type:   is the value type (bool, uint32_t etc)
     *  status: is the current _usage_ of the opt. I.e. if you actually use the value, set it to "Used".
     *          Most values are set to "Unused", as in "will probably have an effect eventually".
     *          Values set to "Invalid" have no meaning/usage in urchin, and should (and will currently)
     *          be signaled to a user providing a config with them, that these settings are pointless.
     *  desc:   documentation.
     *  value...: enumerated valid values if any. Not currently used, but why not...
     *
     *
     * Note:
     * Only values marked as "Used" will be added as command line options.
     * Options marked as "Invalid" will be warned about if read from config
     * Options marked as "Unused" will also warned about.
     *
     */

#define _make_config_values(val)                \
    /* Initialization properties */             \
    /* The minimal properties needed for configuring a cluster. */  \
    val(cluster_name, sstring, "Test Cluster", Unused,   \
            "The name of the cluster; used to prevent machines in one logical cluster from joining another. All nodes participating in a cluster must have the same value."   \
    )                                           \
    val(listen_address, sstring, "localhost", Used,     \
            "The IP address or hostname that Cassandra binds to for connecting to other Cassandra nodes. Set this parameter or listen_interface, not both. You must change the default setting for multiple nodes to communicate:\n"    \
            "\n"    \
            "Generally set to empty. If the node is properly configured (host name, name resolution, and so on), Cassandra uses InetAddress.getLocalHost() to get the local address from the system.\n" \
            "For a single node cluster, you can use the default setting (localhost).\n" \
            "If Cassandra can't find the correct address, you must specify the IP address or host name.\n"  \
            "Never specify 0.0.0.0; it is always wrong."  \
    )                                                   \
    val(listen_interface, sstring, "eth0", Unused,  \
            "The interface that Cassandra binds to for connecting to other Cassandra nodes. Interfaces must correspond to a single address, IP aliasing is not supported. See listen_address."  \
    )   \
    /* Default directories */   \
    /* If you have changed any of the default directories during installation, make sure you have root access and set these properties: */  \
    val(commitlog_directory, sstring, "/var/lib/cassandra/commitlog", Used,   \
            "The directory where the commit log is stored. For optimal write performance, it is recommended the commit log be on a separate disk partition (ideally, a separate physical device) from the data file directories."   \
    )                                           \
    val(data_file_directories, string_list, { "/var/lib/cassandra/data" }, Used,   \
            "The directory location where table data (SSTables) is stored"   \
    )                                           \
    val(saved_caches_directory, sstring, "/var/lib/cassandra/saved_caches", Unused, \
            "The directory location where table key and row caches are stored."  \
    )                                                   \
    /* Commonly used properties */  \
    /* Properties most frequently used when configuring Cassandra. */   \
    /* Before starting a node for the first time, you should carefully evaluate your requirements. */   \
    /* Common initialization properties */  \
    /* Note: Be sure to set the properties in the Quick start section as well. */   \
    val(commit_failure_policy, sstring, "stop", Unused, \
            "Policy for commit disk failures:\n"    \
            "\n"    \
            "\tdie          Shut down gossip and Thrift and kill the JVM, so the node can be replaced.\n"   \
            "\tstop         Shut down gossip and Thrift, leaving the node effectively dead, but can be inspected using JMX.\n" \
            "\tstop_commit  Shut down the commit log, letting writes collect but continuing to service reads (as in pre-2.0.5 Cassandra).\n"    \
            "\tignore       Ignore fatal errors and let the batches fail."\
            , "die", "stop", "stop_commit", "ignore"    \
    )   \
    val(disk_failure_policy, sstring, "stop", Unused, \
            "Sets how Cassandra responds to disk failure. Recommend settings are stop or best_effort.\n"    \
            "\n"    \
            "\tdie              Shut down gossip and Thrift and kill the JVM for any file system errors or single SSTable errors, so the node can be replaced.\n"   \
            "\tstop_paranoid    Shut down gossip and Thrift even for single SSTable errors.\n"    \
            "\tstop             Shut down gossip and Thrift, leaving the node effectively dead, but available for inspection using JMX.\n"    \
            "\tbest_effort      Stop using the failed disk and respond to requests based on the remaining available SSTables. This means you will see obsolete data at consistency level of ONE.\n"    \
            "\tignore           Ignores fatal errors and lets the requests fail; all file system errors are logged but otherwise ignored. Cassandra acts as in versions prior to 1.2.\n"    \
            "\n"    \
            "Related information: Handling Disk Failures In Cassandra 1.2 blog and Recovering from a single disk failure using JBOD.\n"    \
            , "die", "stop_paranoid", "stop", "best_effort", "ignore"   \
    )   \
    val(endpoint_snitch, sstring, "org.apache.cassandra.locator.SimpleSnitch", Used,  \
            "Set to a class that implements the IEndpointSnitch. Cassandra uses snitches for locating nodes and routing requests.\n"    \
            "\tSimpleSnitch                 Use for single-data center deployments or single-zone in public clouds. Does not recognize data center or rack information. It treats strategy order as proximity, which can improve cache locality when disabling read repair.\n"    \
            "\tGossipingPropertyFileSnitch  Recommended for production. The rack and data center for the local node are defined in the cassandra-rackdc.properties file and propagated to other nodes via gossip. To allow migration from the PropertyFileSnitch, it uses the cassandra-topology.properties file if it is present.\n"    \
            "\tPropertyFileSnitch           Determines proximity by rack and data center, which are explicitly configured in the cassandra-topology.properties file.\n"    \
            "\tEc2Snitch                    For EC2 deployments in a single region. Loads region and availability zone information from the EC2 API. The region is treated as the data center and the availability zone as the rack. Uses only private IPs. Subsequently it does not work across multiple regions.\n"    \
            "\tEc2MultiRegionSnitch         Uses public IPs as the broadcast_address to allow cross-region connectivity. This means you must also set seed addresses to the public IP and open the storage_port or ssl_storage_port on the public IP firewall. For intra-region traffic, Cassandra switches to the private IP after establishing a connection.\n"    \
            "\tRackInferringSnitch          Proximity is determined by rack and data center, which are assumed to correspond to the 3rd and 2nd octet of each node's IP address, respectively. This snitch is best used as an example for writing a custom snitch class (unless this happens to match your deployment conventions).\n" \
            "\n"    \
            "Related information: Snitches\n"    \
    )                                                   \
    val(rpc_address, sstring, "localhost", Used,     \
            "The listen address for client connections (Thrift RPC service and native transport).Valid values are:\n"    \
            "\n"    \
            "\tunset:   Resolves the address using the hostname configuration of the node. If left unset, the hostname must resolve to the IP address of this node using /etc/hostname, /etc/hosts, or DNS.\n"  \
            "\t0.0.0.0 : Listens on all configured interfaces, but you must set the broadcast_rpc_address to a value other than 0.0.0.0.\n"   \
            "\tIP address\n"    \
            "\thostname\n"  \
            "Related information: Network\n"    \
    )                                                   \
    val(rpc_interface, sstring, "eth1", Unused,     \
            "The listen address for client connections. Interfaces must correspond to a single address, IP aliasing is not supported. See rpc_address." \
    )   \
    val(seed_provider, seed_provider_type, seed_provider_type("org.apache.cassandra.locator.SimpleSeedProvider"), Used, \
            "The addresses of hosts deemed contact points. Cassandra nodes use the -seeds list to find each other and learn the topology of the ring.\n"    \
            "\n"    \
            "  class_name (Default: org.apache.cassandra.locator.SimpleSeedProvider)\n" \
            "  \tThe class within Cassandra that handles the seed logic. It can be customized, but this is typically not required.\n"   \
            "  \t- seeds (Default: 127.0.0.1)    A comma-delimited list of IP addresses used by gossip for bootstrapping new nodes joining a cluster. When running multiple nodes, you must change the list from the default value. In multiple data-center clusters, the seed list should include at least one node from each data center (replication group). More than a single seed node per data center is recommended for fault tolerance. Otherwise, gossip has to communicate with another data center when bootstrapping a node. Making every node a seed node is not recommended because of increased maintenance and reduced gossip performance. Gossip optimization is not critical, but it is recommended to use a small seed list (approximately three nodes per data center).\n"    \<commit_msg>db::config: improve a help output for --endpoint_snitch parameter<commit_after>/*
 * Copyright 2015 Cloudius Systems
 *
 */

#pragma once

#include <boost/program_options.hpp>
#include <unordered_map>
#include "core/sstring.hh"
#include "core/future.hh"

class file;

namespace db {

typedef std::unordered_map<sstring, sstring> string_map;

/*
 * This type is not use, and probably never will be.
 * So it makes sense to jump through hoops just to ensure
 * it is in fact handled properly...
 */
struct seed_provider_type {
    seed_provider_type() = default;
    seed_provider_type(sstring n,
            std::initializer_list<string_map::value_type> opts =
                    { })
            : class_name(std::move(n)), parameters(std::move(opts)) {
    }
    sstring class_name;
    string_map parameters;
};

class config {
public:
    enum class value_status {
        Used,
        Unused,
        Invalid,
    };

    enum class config_source : uint8_t {
        None,
        SettingsFile,
        CommandLine
    };

    template<typename T, value_status S>
    struct value {
        typedef T type;
        typedef value<T, S> MyType;

        value(const T& t = T()) : _value(t)
        {}
        value_status status() const {
            return S;
        }
        config_source source() const {
            return _source;
        }
        bool is_set() const {
            return _source > config_source::None;
        }
        MyType & operator()(const T& t) {
            _value = t;
            return *this;
        }
        MyType & operator()(T&& t) {
            _value = std::move(t);
            return *this;
        }
        const T& operator()() const {
            return _value;
        }
        T& operator()() {
            return _value;
        }
    private:
        friend class config;
        T _value = T();
        config_source _source = config_source::None;
    };

    config();

    boost::program_options::options_description
    get_options_description();

    boost::program_options::options_description_easy_init&
    add_options(boost::program_options::options_description_easy_init&);

    void read_from_yaml(const sstring&);
    void read_from_yaml(const char *);
    future<> read_from_file(const sstring&);
    future<> read_from_file(file);

    typedef std::unordered_map<sstring, sstring> string_map;
    typedef std::vector<sstring> string_list;
    using seed_provider_type = db::seed_provider_type;

    /*
     * All values and documentation taken from
     * http://docs.datastax.com/en/cassandra/2.1/cassandra/configuration/configCassandra_yaml_r.html
     *
     * Big fat x-macro expansion of _all_ (at this writing) cassandra opts, with the "documentation"
     * included.
     *
     * X-macro syntax:
     *
     * X(member, type, default, status, desc [, value, value...])
     *
     * Where
     *  member: is the property name -> config member name
     *  type:   is the value type (bool, uint32_t etc)
     *  status: is the current _usage_ of the opt. I.e. if you actually use the value, set it to "Used".
     *          Most values are set to "Unused", as in "will probably have an effect eventually".
     *          Values set to "Invalid" have no meaning/usage in urchin, and should (and will currently)
     *          be signaled to a user providing a config with them, that these settings are pointless.
     *  desc:   documentation.
     *  value...: enumerated valid values if any. Not currently used, but why not...
     *
     *
     * Note:
     * Only values marked as "Used" will be added as command line options.
     * Options marked as "Invalid" will be warned about if read from config
     * Options marked as "Unused" will also warned about.
     *
     */

#define _make_config_values(val)                \
    /* Initialization properties */             \
    /* The minimal properties needed for configuring a cluster. */  \
    val(cluster_name, sstring, "Test Cluster", Unused,   \
            "The name of the cluster; used to prevent machines in one logical cluster from joining another. All nodes participating in a cluster must have the same value."   \
    )                                           \
    val(listen_address, sstring, "localhost", Used,     \
            "The IP address or hostname that Cassandra binds to for connecting to other Cassandra nodes. Set this parameter or listen_interface, not both. You must change the default setting for multiple nodes to communicate:\n"    \
            "\n"    \
            "Generally set to empty. If the node is properly configured (host name, name resolution, and so on), Cassandra uses InetAddress.getLocalHost() to get the local address from the system.\n" \
            "For a single node cluster, you can use the default setting (localhost).\n" \
            "If Cassandra can't find the correct address, you must specify the IP address or host name.\n"  \
            "Never specify 0.0.0.0; it is always wrong."  \
    )                                                   \
    val(listen_interface, sstring, "eth0", Unused,  \
            "The interface that Cassandra binds to for connecting to other Cassandra nodes. Interfaces must correspond to a single address, IP aliasing is not supported. See listen_address."  \
    )   \
    /* Default directories */   \
    /* If you have changed any of the default directories during installation, make sure you have root access and set these properties: */  \
    val(commitlog_directory, sstring, "/var/lib/cassandra/commitlog", Used,   \
            "The directory where the commit log is stored. For optimal write performance, it is recommended the commit log be on a separate disk partition (ideally, a separate physical device) from the data file directories."   \
    )                                           \
    val(data_file_directories, string_list, { "/var/lib/cassandra/data" }, Used,   \
            "The directory location where table data (SSTables) is stored"   \
    )                                           \
    val(saved_caches_directory, sstring, "/var/lib/cassandra/saved_caches", Unused, \
            "The directory location where table key and row caches are stored."  \
    )                                                   \
    /* Commonly used properties */  \
    /* Properties most frequently used when configuring Cassandra. */   \
    /* Before starting a node for the first time, you should carefully evaluate your requirements. */   \
    /* Common initialization properties */  \
    /* Note: Be sure to set the properties in the Quick start section as well. */   \
    val(commit_failure_policy, sstring, "stop", Unused, \
            "Policy for commit disk failures:\n"    \
            "\n"    \
            "\tdie          Shut down gossip and Thrift and kill the JVM, so the node can be replaced.\n"   \
            "\tstop         Shut down gossip and Thrift, leaving the node effectively dead, but can be inspected using JMX.\n" \
            "\tstop_commit  Shut down the commit log, letting writes collect but continuing to service reads (as in pre-2.0.5 Cassandra).\n"    \
            "\tignore       Ignore fatal errors and let the batches fail."\
            , "die", "stop", "stop_commit", "ignore"    \
    )   \
    val(disk_failure_policy, sstring, "stop", Unused, \
            "Sets how Cassandra responds to disk failure. Recommend settings are stop or best_effort.\n"    \
            "\n"    \
            "\tdie              Shut down gossip and Thrift and kill the JVM for any file system errors or single SSTable errors, so the node can be replaced.\n"   \
            "\tstop_paranoid    Shut down gossip and Thrift even for single SSTable errors.\n"    \
            "\tstop             Shut down gossip and Thrift, leaving the node effectively dead, but available for inspection using JMX.\n"    \
            "\tbest_effort      Stop using the failed disk and respond to requests based on the remaining available SSTables. This means you will see obsolete data at consistency level of ONE.\n"    \
            "\tignore           Ignores fatal errors and lets the requests fail; all file system errors are logged but otherwise ignored. Cassandra acts as in versions prior to 1.2.\n"    \
            "\n"    \
            "Related information: Handling Disk Failures In Cassandra 1.2 blog and Recovering from a single disk failure using JBOD.\n"    \
            , "die", "stop_paranoid", "stop", "best_effort", "ignore"   \
    )   \
    val(endpoint_snitch, sstring, "org.apache.cassandra.locator.SimpleSnitch", Used,  \
            "Set to a class that implements the IEndpointSnitch. Cassandra uses snitches for locating nodes and routing requests.\n\n"    \
            "\tSimpleSnitch: Use for single-data center deployments or single-zone in public clouds. Does not recognize data center or rack information. It treats strategy order as proximity, which can improve cache locality when disabling read repair.\n\n"    \
            "\tGossipingPropertyFileSnitch: Recommended for production. The rack and data center for the local node are defined in the cassandra-rackdc.properties file and propagated to other nodes via gossip. To allow migration from the PropertyFileSnitch, it uses the cassandra-topology.properties file if it is present.\n\n"    \
            /*"\tPropertyFileSnitch: Determines proximity by rack and data center, which are explicitly configured in the cassandra-topology.properties file.\n\n"    */\
            /*"\tEc2Snitch: For EC2 deployments in a single region. Loads region and availability zone information from the EC2 API. The region is treated as the data center and the availability zone as the rack. Uses only private IPs. Subsequently it does not work across multiple regions.\n\n"    */\
            /*"\tEc2MultiRegionSnitch: Uses public IPs as the broadcast_address to allow cross-region connectivity. This means you must also set seed addresses to the public IP and open the storage_port or ssl_storage_port on the public IP firewall. For intra-region traffic, Cassandra switches to the private IP after establishing a connection.\n\n"    */\
            "\tRackInferringSnitch: Proximity is determined by rack and data center, which are assumed to correspond to the 3rd and 2nd octet of each node's IP address, respectively. This snitch is best used as an example for writing a custom snitch class (unless this happens to match your deployment conventions).\n" \
            "\n"    \
            "Related information: Snitches\n"    \
    )                                                   \
    val(rpc_address, sstring, "localhost", Used,     \
            "The listen address for client connections (Thrift RPC service and native transport).Valid values are:\n"    \
            "\n"    \
            "\tunset:   Resolves the address using the hostname configuration of the node. If left unset, the hostname must resolve to the IP address of this node using /etc/hostname, /etc/hosts, or DNS.\n"  \
            "\t0.0.0.0 : Listens on all configured interfaces, but you must set the broadcast_rpc_address to a value other than 0.0.0.0.\n"   \
            "\tIP address\n"    \
            "\thostname\n"  \
            "Related information: Network\n"    \
    )                                                   \
    val(rpc_interface, sstring, "eth1", Unused,     \
            "The listen address for client connections. Interfaces must correspond to a single address, IP aliasing is not supported. See rpc_address." \
    )   \
    val(seed_provider, seed_provider_type, seed_provider_type("org.apache.cassandra.locator.SimpleSeedProvider"), Used, \
            "The addresses of hosts deemed contact points. Cassandra nodes use the -seeds list to find each other and learn the topology of the ring.\n"    \
            "\n"    \
            "  class_name (Default: org.apache.cassandra.locator.SimpleSeedProvider)\n" \
            "  \tThe class within Cassandra that handles the seed logic. It can be customized, but this is typically not required.\n"   \
            "  \t- seeds (Default: 127.0.0.1)    A comma-delimited list of IP addresses used by gossip for bootstrapping new nodes joining a cluster. When running multiple nodes, you must change the list from the default value. In multiple data-center clusters, the seed list should include at least one node from each data center (replication group). More than a single seed node per data center is recommended for fault tolerance. Otherwise, gossip has to communicate with another data center when bootstrapping a node. Making every node a seed node is not recommended because of increased maintenance and reduced gossip performance. Gossip optimization is not critical, but it is recommended to use a small seed list (approximately three nodes per data center).\n"    \
{'fraction_non_alphanumeric': 0.07681558824671039, 'fraction_numerical': 0.003260489849784575, 'mean_word_length': 2.2699581165122478, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5058560', 'n_tokens_mistral': 6768, 'n_tokens_neox': 6547, 'n_words': 2769}