<commit_before>package eu.modernmt.cli;

import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import eu.modernmt.cli.log4j.Log4jConfiguration;
import eu.modernmt.cluster.ClusterNode;
import eu.modernmt.cluster.EmbeddedService;
import eu.modernmt.config.*;
import eu.modernmt.config.xml.XMLConfigBuilder;
import eu.modernmt.engine.Engine;
import eu.modernmt.facade.ModernMT;
import eu.modernmt.io.UTF8Charset;
import org.apache.commons.cli.*;
import org.apache.commons.io.FileUtils;

import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.List;

/**
 * Created by davide on 22/04/16.
 * Updated by andrearossi on 18/04/16.
 */
public class ClusterNodeMain {

    /**
     * This class models the list of arguments that can be passed to the ClusterNodeMain main.
     * It is typically created by parsing the arguments passed from the outside during the execution of MMT start.
     */
    private static class Args {

        /* Define the name and types of the arguments as Option objects*/
        private static final Options cliOptions;

        static {
            Option engine = Option.builder("e").longOpt("engine").hasArg().required().build();
            Option statusFile = Option.builder().longOpt("status-file").hasArg().required().build();
            Option logFile = Option.builder().longOpt("log-file").hasArg().required().build();

            Option apiPort = Option.builder("a").longOpt("api-port").hasArg().type(Integer.class).required(false).build();
            Option clusterPort = Option.builder("p").longOpt("cluster-port").hasArg().type(Integer.class).required(false).build();
            Option datastreamPort = Option.builder().longOpt("datastream-port").hasArg().required(false).build();
            Option databasePort = Option.builder().longOpt("cassandra-port").hasArg().required(false).build();

            Option leader = Option.builder().longOpt("leader").hasArg().required(false).build();

            Option verbosity = Option.builder("v").longOpt("verbosity").hasArg().type(Integer.class).required(false).build();

            cliOptions = new Options();
            cliOptions.addOption(engine);
            cliOptions.addOption(apiPort);
            cliOptions.addOption(clusterPort);
            cliOptions.addOption(statusFile);
            cliOptions.addOption(verbosity);
            cliOptions.addOption(leader);
            cliOptions.addOption(logFile);
            cliOptions.addOption(datastreamPort);
            cliOptions.addOption(databasePort);
        }

        public final String engine;
        public final File statusFile;
        public final File logFile;
        public final int verbosity;
        public final NodeConfig config;

        /**
         * This builder parses the command line arguments passed by the Python module
         * into the Options object statically defined by Args.
         * After that, it reads the various options from the Options object
         * and stores them in the corresponding instance variables.
         * <p>
         * Finally, it uses the obtained options to create a new NodeConfig.
         * The Nodeconfig is initially generated by parsing the engine.xconf file,
         * and then its values are overwritten if necessary with the cli values
         *
         * @param args the arguments passed by the Python module
         *             when invoking the ClusterNodeMain main
         * @throws ParseException
         * @throws ConfigException
         */
        public Args(String[] args) throws ParseException, ConfigException {
            CommandLineParser parser = new DefaultParser();
            CommandLine cli = parser.parse(cliOptions, args);

            this.engine = cli.getOptionValue("engine");
            this.statusFile = new File(cli.getOptionValue("status-file"));
            this.logFile = new File(cli.getOptionValue("log-file"));

            String verbosity = cli.getOptionValue("verbosity");
            this.verbosity = verbosity == null ? 1 : Integer.parseInt(verbosity);

            /* read the engine.xconf file and build the engineConfig, with its:
               network config, datastream config, database config, net-api config and net-join config */
            this.config = XMLConfigBuilder.build(Engine.getConfigFile(this.engine));
            this.config.getEngineConfig().setName(this.engine);

            // Create the config objects based on the engine.xconf file
            NetworkConfig netConfig = this.config.getNetworkConfig();
            DataStreamConfig streamConfig = this.config.getDataStreamConfig();
            DatabaseConfig dbConfig = this.config.getDatabaseConfig();
            ApiConfig apiConfig = netConfig.getApiConfig();
            JoinConfig joinConfig = netConfig.getJoinConfig();

            /* alter the values read from the engine.xconf file in the following cases:*/

            // ~~~~~~~~~~~~~~~~ PORTS ~~~~~~~~~~~~~~~~~~~
            /* Ports passed by command line have priority over the ones read by engine.xconf
            (that may be either the default ones or actual values written by the user) so use them if they are not null.*/
            String port = cli.getOptionValue("cluster-port");
            if (port != null)
                netConfig.setPort(Integer.parseInt(port));
            String apiPort = cli.getOptionValue("api-port");
            if (apiPort != null)
                apiConfig.setPort(Integer.parseInt(apiPort));
            String datastreamPort = cli.getOptionValue("datastream-port");
            if (datastreamPort != null)
                streamConfig.setPort(Integer.parseInt(datastreamPort));
            String databasePort = cli.getOptionValue("cassandra-port");<commit_msg>bug fixed: changed the name of parameter for setting the database port<commit_after>package eu.modernmt.cli;

import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import eu.modernmt.cli.log4j.Log4jConfiguration;
import eu.modernmt.cluster.ClusterNode;
import eu.modernmt.cluster.EmbeddedService;
import eu.modernmt.config.*;
import eu.modernmt.config.xml.XMLConfigBuilder;
import eu.modernmt.engine.Engine;
import eu.modernmt.facade.ModernMT;
import eu.modernmt.io.UTF8Charset;
import org.apache.commons.cli.*;
import org.apache.commons.io.FileUtils;

import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.List;

/**
 * Created by davide on 22/04/16.
 * Updated by andrearossi on 18/04/16.
 */
public class ClusterNodeMain {

    /**
     * This class models the list of arguments that can be passed to the ClusterNodeMain main.
     * It is typically created by parsing the arguments passed from the outside during the execution of MMT start.
     */
    private static class Args {

        /* Define the name and types of the arguments as Option objects*/
        private static final Options cliOptions;

        static {
            Option engine = Option.builder("e").longOpt("engine").hasArg().required().build();
            Option statusFile = Option.builder().longOpt("status-file").hasArg().required().build();
            Option logFile = Option.builder().longOpt("log-file").hasArg().required().build();

            Option apiPort = Option.builder("a").longOpt("api-port").hasArg().type(Integer.class).required(false).build();
            Option clusterPort = Option.builder("p").longOpt("cluster-port").hasArg().type(Integer.class).required(false).build();
            Option datastreamPort = Option.builder().longOpt("datastream-port").hasArg().required(false).build();
            Option databasePort = Option.builder().longOpt("db-port").hasArg().required(false).build();

            Option leader = Option.builder().longOpt("leader").hasArg().required(false).build();

            Option verbosity = Option.builder("v").longOpt("verbosity").hasArg().type(Integer.class).required(false).build();

            cliOptions = new Options();
            cliOptions.addOption(engine);
            cliOptions.addOption(apiPort);
            cliOptions.addOption(clusterPort);
            cliOptions.addOption(statusFile);
            cliOptions.addOption(verbosity);
            cliOptions.addOption(leader);
            cliOptions.addOption(logFile);
            cliOptions.addOption(datastreamPort);
            cliOptions.addOption(databasePort);
        }

        public final String engine;
        public final File statusFile;
        public final File logFile;
        public final int verbosity;
        public final NodeConfig config;

        /**
         * This builder parses the command line arguments passed by the Python module
         * into the Options object statically defined by Args.
         * After that, it reads the various options from the Options object
         * and stores them in the corresponding instance variables.
         * <p>
         * Finally, it uses the obtained options to create a new NodeConfig.
         * The Nodeconfig is initially generated by parsing the engine.xconf file,
         * and then its values are overwritten if necessary with the cli values
         *
         * @param args the arguments passed by the Python module
         *             when invoking the ClusterNodeMain main
         * @throws ParseException
         * @throws ConfigException
         */
        public Args(String[] args) throws ParseException, ConfigException {
            CommandLineParser parser = new DefaultParser();
            CommandLine cli = parser.parse(cliOptions, args);

            this.engine = cli.getOptionValue("engine");
            this.statusFile = new File(cli.getOptionValue("status-file"));
            this.logFile = new File(cli.getOptionValue("log-file"));

            String verbosity = cli.getOptionValue("verbosity");
            this.verbosity = verbosity == null ? 1 : Integer.parseInt(verbosity);

            /* read the engine.xconf file and build the engineConfig, with its:
               network config, datastream config, database config, net-api config and net-join config */
            this.config = XMLConfigBuilder.build(Engine.getConfigFile(this.engine));
            this.config.getEngineConfig().setName(this.engine);

            // Create the config objects based on the engine.xconf file
            NetworkConfig netConfig = this.config.getNetworkConfig();
            DataStreamConfig streamConfig = this.config.getDataStreamConfig();
            DatabaseConfig dbConfig = this.config.getDatabaseConfig();
            ApiConfig apiConfig = netConfig.getApiConfig();
            JoinConfig joinConfig = netConfig.getJoinConfig();

            /* alter the values read from the engine.xconf file in the following cases:*/

            // ~~~~~~~~~~~~~~~~ PORTS ~~~~~~~~~~~~~~~~~~~
            /* Ports passed by command line have priority over the ones read by engine.xconf
            (that may be either the default ones or actual values written by the user) so use them if they are not null.*/
            String port = cli.getOptionValue("cluster-port");
            if (port != null)
                netConfig.setPort(Integer.parseInt(port));
            String apiPort = cli.getOptionValue("api-port");
            if (apiPort != null)
                apiConfig.setPort(Integer.parseInt(apiPort));
            String datastreamPort = cli.getOptionValue("datastream-port");
            if (datastreamPort != null)
                streamConfig.setPort(Integer.parseInt(datastreamPort));
            String databasePort = cli.getOptionValue("db-port");
{'fraction_non_alphanumeric': 0.10651142733937041, 'fraction_numerical': 0.002759810263044416, 'mean_word_length': 2.7238278741168913, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3633797', 'n_tokens_mistral': 2975, 'n_tokens_neox': 2874, 'n_words': 812}