<commit_before>/**
 *    Retz
 *    Copyright (C) 2016-2017 Nautilus Technologies, Inc.
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */
package io.github.retz.scheduler;

import io.github.retz.cli.FileConfiguration;
import io.github.retz.protocol.data.ResourceQuantity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;

public class ServerConfiguration extends FileConfiguration {

    // System Limits
    public static final String MAX_SIMULTANEOUS_JOBS = "retz.max.running";
    public static final String DEFAULT_MAX_SIMULTANEOUS_JOBS = "128";
    public static final String MAX_STOCK_SIZE = "retz.max.stock";
    public static final String DEFAULT_MAX_STOCK_SIZE = "16";
    public static final String MAX_CPUS = "retz.max.cpus";
    public static final String DEFAULT_MAX_CPUS = "8";
    public static final String MAX_MEM = "retz.max.mem";
    public static final String DEFAULT_MAX_MEM = "31744"; // (32 - 1) * 1024 MB
    public static final String MAX_GPUS = "retz.max.gpus";
    public static final String DEFAULT_MAX_GPUS = "0";
    public static final String MAX_PORTS = "retz.max.ports";
    public static final String DEFAULT_MAX_PORTS = "10";
    public static final String MAX_DISK = "retz.max.disk";
    public static final String DEFAULT_MAX_DISK = "1024"; // in MB
    public static final String MAX_LIST_JOB_SIZE = "retz.max.list-jobs";
    public static final String DEFAULT_MAX_LIST_JOB_SIZE = "65536";
    public static final String MAX_FILE_SIZE = "retz.max.file-size";
    public static final String DEFAULT_MAX_FILE_SIZE = Long.toString(65536 * 1024); // in bytes

    // Mesos connections and so on
    static final String MESOS_LOC_KEY = "retz.mesos";
    // TODO: Sparkjava (http://sparkjava.com) only binds to 0.0.0.0, but it must be any IP address
    static final String BIND_ADDRESS = "retz.bind";
    static final String MESOS_ROLE = "retz.mesos.role";
    static final String MESOS_PRINCIPAL = "retz.mesos.principal";
    static final String DEFAULT_MESOS_PRINCIPAL = "retz";
    static final String MESOS_SECRET_FILE = "retz.mesos.secret.file";
    static final String MESOS_REFUSE_SECONDS = "retz.mesos.refuse";
    static final int DEFAULT_MESOS_REFUSE_SECONDS = 3;
    static final String MESOS_FAILOVER_TIMEOUT = "retz.mesos.failover.timeout";
    static final int DEFAULT_MESOS_FAILOVER_TIMEOUT = 3600 * 24 * 7;

    // Not yet used
    static final String QUEUE_MAX = "retz.max.queue";
    static final String SCHEDULE_RESULTS = "retz.results";
    static final String SCHEDULE_RETRY = "retz.retry";
    // Persistence
    static final String DATABASE_URL = "retz.database.url";
    static final String DEFAULT_DATABASE_URL = "jdbc:h2:mem:retz-server;DB_CLOSE_DELAY=-1";
    static final String DATABASE_DRIVER_CLASS = "retz.database.driver";
    static final String DEFAULT_DATABASE_DRIVER_CLASS = "org.h2.Driver";
    static final String DATABASE_USERNAME = "retz.database.user";
    static final String DATABASE_PASSWORD = "<PASSWORD>";
    // https://github.com/apache/mesos/blob/master/include/mesos/mesos.proto#L208-L210
    static final String USER_NAME = "retz.user";
    // static final String[] INVALID_BIND_ADDRESS = {"0.0.0.0", "localhost", "127.0.0.1"};
    private static final Logger LOG = LoggerFactory.getLogger(ServerConfiguration.class);
    private final URI uri;
    private final int maxSimultaneousJobs;
    private final String databaseURL;
    private final String databaseDriver;

    private static final String PLANNER_NAME = "retz.planner.name";
    private static final String DEFAULT_PLANNER_NAME = "fifo";
    private static final String[] PLANNER_NAMES = {"naive", "priority", "fifo", "priority2"};

    private static final String JOB_QUEUE_STRATEGY = "retz.job-queue-strategy";
    private static final String DEFAULT_JOB_QUEUE_STRATEGY = "all";
    public enum JobQueueStrategy {
        Fit,
        All;
        static JobQueueStrategy getStrategy(String s) {
            switch(s) {
                case "all": return JobQueueStrategy.All;
                case "fit": return JobQueueStrategy.Fit;
                default: return null;
            }
        }
    }

    private static final String ADDITIONAL_CLASSPATH = "retz.classpath";
    private static final String DEFAULT_ADDITIONAL_CLASSPATH = "/opt/retz-server/lib";

    // Leeway seconds before old job entries get deleted by Retz.
    // As we may think of many race conditions on task finish at Mesos
    // and retry from Retz, updates, this value must be reasonably
    // large number, like several days that old tasks my not reincarnate.
    // For example, if a cluster usage accounting batch is scheduled
    // every week, this must be long enough that *all* jobs are
    // accounted and not deleted before any batch.
    private static final String GC = "retz.gc";
    private static final String GC_LEEWAY = "retz.gc.leeway";
    private static final int DEFAULT_GC_LEEWAY = 7 * 86400; // a week in seconds
    private static final String GC_INTERVAL = "retz.gc.interval";
    private static final int DEFAULT_GC_INTERVAL = 600; // 10 minutes in seconds


    public ServerConfiguration(InputStream in) throws IOException, URISyntaxException {
        super(in);


        Objects.requireNonNull(properties.getProperty(BIND_ADDRESS), "Host and port are required");
        Objects.requireNonNull(getMesosMaster(), "Mesos address must not be empty");

        uri = new URI(properties.getProperty(BIND_ADDRESS));
        // List<String> invalidBindAddresses = Arrays.asList(INVALID_BIND_ADDRESS);
        // loopback addresses must also be checked, but for now inttest uses localhost address
        if (uri.getHost().equals("0.0.0.0")) {
            LOG.error("retz.bind is told to Mesos; {}/32 should not be assigned", uri.getHost());
            throw new IllegalArgumentException();
        }
        if (uri.getPort() < 1024 || 65536 < uri.getPort()) {
            LOG.error("retz.bind must not use well known port, or just too large: {}", uri.getPort());
            throw new IllegalArgumentException();
        }

        maxSimultaneousJobs = Integer.parseInt(properties.getProperty(MAX_SIMULTANEOUS_JOBS, DEFAULT_MAX_SIMULTANEOUS_JOBS));
        if (maxSimultaneousJobs < 1) {
            throw new IllegalArgumentException(MAX_SIMULTANEOUS_JOBS + " must be positive");
        }

        databaseURL = properties.getProperty(DATABASE_URL, DEFAULT_DATABASE_URL);
        databaseDriver = properties.getProperty(DATABASE_DRIVER_CLASS, DEFAULT_DATABASE_DRIVER_CLASS);

        if ("root".equals(getUserName()) || getUserName().isEmpty()) {
            LOG.error("{} must not be 'root' nor empty", USER_NAME);
            throw new IllegalArgumentException("Invalid parameter: " + USER_NAME);
        }

        if (getRefuseSeconds() < 1) {
            throw new IllegalArgumentException(MESOS_REFUSE_SECONDS + " must be positive integer");
        }

        if (getJobQueueStrategy() == null) {
            throw new IllegalArgumentException(JOB_QUEUE_STRATEGY + " must be either fir or all");
        }

        LOG.info("Mesos master={}, principal={}, role={}, {}={}, {}={}, {}={}, {}={}, {}={}, {}={}, {}={}, {}={}, {}={}",
                getMesosMaster(), getPrincipal(), getRole(), MAX_SIMULTANEOUS_JOBS, maxSimultaneousJobs,
                DATABASE_URL, databaseURL,
                MAX_STOCK_SIZE, getMaxStockSize(),
                USER_NAME, getUserName(),
                MESOS_REFUSE_SECONDS, getRefuseSeconds(),
                GC_LEEWAY, getGcLeeway(),
                GC_INTERVAL, getGcInterval(),
                MAX_LIST_JOB_SIZE, getMaxJobSize(),
                MAX_FILE_SIZE, getMaxFileSize());
        LOG.info("{}={}", MESOS_FAILOVER_TIMEOUT, getFailoverTimeout());
    }

    public ServerConfiguration(String file) throws IOException, URISyntaxException {
        this(new FileInputStream(file));
    }

    // This is not only for Mesos master address, but mainly for ZooKeeper location
    // that enables Mesos master service discovery opaquely over failure.
    public String getMesosMaster() {
        return properties.getProperty(MESOS_LOC_KEY);
    }

    public URI getUri() {
        return uri;<commit_msg>Print job queue strategy on start<commit_after>/**
 *    Retz
 *    Copyright (C) 2016-2017 Nautilus Technologies, Inc.
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */
package io.github.retz.scheduler;

import io.github.retz.cli.FileConfiguration;
import io.github.retz.protocol.data.ResourceQuantity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;

public class ServerConfiguration extends FileConfiguration {

    // System Limits
    public static final String MAX_SIMULTANEOUS_JOBS = "retz.max.running";
    public static final String DEFAULT_MAX_SIMULTANEOUS_JOBS = "128";
    public static final String MAX_STOCK_SIZE = "retz.max.stock";
    public static final String DEFAULT_MAX_STOCK_SIZE = "16";
    public static final String MAX_CPUS = "retz.max.cpus";
    public static final String DEFAULT_MAX_CPUS = "8";
    public static final String MAX_MEM = "retz.max.mem";
    public static final String DEFAULT_MAX_MEM = "31744"; // (32 - 1) * 1024 MB
    public static final String MAX_GPUS = "retz.max.gpus";
    public static final String DEFAULT_MAX_GPUS = "0";
    public static final String MAX_PORTS = "retz.max.ports";
    public static final String DEFAULT_MAX_PORTS = "10";
    public static final String MAX_DISK = "retz.max.disk";
    public static final String DEFAULT_MAX_DISK = "1024"; // in MB
    public static final String MAX_LIST_JOB_SIZE = "retz.max.list-jobs";
    public static final String DEFAULT_MAX_LIST_JOB_SIZE = "65536";
    public static final String MAX_FILE_SIZE = "retz.max.file-size";
    public static final String DEFAULT_MAX_FILE_SIZE = Long.toString(65536 * 1024); // in bytes

    // Mesos connections and so on
    static final String MESOS_LOC_KEY = "retz.mesos";
    // TODO: Sparkjava (http://sparkjava.com) only binds to 0.0.0.0, but it must be any IP address
    static final String BIND_ADDRESS = "retz.bind";
    static final String MESOS_ROLE = "retz.mesos.role";
    static final String MESOS_PRINCIPAL = "retz.mesos.principal";
    static final String DEFAULT_MESOS_PRINCIPAL = "retz";
    static final String MESOS_SECRET_FILE = "retz.mesos.secret.file";
    static final String MESOS_REFUSE_SECONDS = "retz.mesos.refuse";
    static final int DEFAULT_MESOS_REFUSE_SECONDS = 3;
    static final String MESOS_FAILOVER_TIMEOUT = "retz.mesos.failover.timeout";
    static final int DEFAULT_MESOS_FAILOVER_TIMEOUT = 3600 * 24 * 7;

    // Not yet used
    static final String QUEUE_MAX = "retz.max.queue";
    static final String SCHEDULE_RESULTS = "retz.results";
    static final String SCHEDULE_RETRY = "retz.retry";
    // Persistence
    static final String DATABASE_URL = "retz.database.url";
    static final String DEFAULT_DATABASE_URL = "jdbc:h2:mem:retz-server;DB_CLOSE_DELAY=-1";
    static final String DATABASE_DRIVER_CLASS = "retz.database.driver";
    static final String DEFAULT_DATABASE_DRIVER_CLASS = "org.h2.Driver";
    static final String DATABASE_USERNAME = "retz.database.user";
    static final String DATABASE_PASSWORD = "<PASSWORD>";
    // https://github.com/apache/mesos/blob/master/include/mesos/mesos.proto#L208-L210
    static final String USER_NAME = "retz.user";
    // static final String[] INVALID_BIND_ADDRESS = {"0.0.0.0", "localhost", "127.0.0.1"};
    private static final Logger LOG = LoggerFactory.getLogger(ServerConfiguration.class);
    private final URI uri;
    private final int maxSimultaneousJobs;
    private final String databaseURL;
    private final String databaseDriver;

    private static final String PLANNER_NAME = "retz.planner.name";
    private static final String DEFAULT_PLANNER_NAME = "fifo";
    private static final String[] PLANNER_NAMES = {"naive", "priority", "fifo", "priority2"};

    private static final String JOB_QUEUE_STRATEGY = "retz.job-queue-strategy";
    private static final String DEFAULT_JOB_QUEUE_STRATEGY = "all";
    public enum JobQueueStrategy {
        Fit("fit"),
        All("all");
        static JobQueueStrategy getStrategy(String s) {
            switch(s.toLowerCase()) {
                case "all": return JobQueueStrategy.All;
                case "fit": return JobQueueStrategy.Fit;
                default: return null;
            }
        }
        private final String text;
        JobQueueStrategy(final String text) {
            this.text = text;
        }
        @Override
        public String toString() {
            return text;
        }
    }

    private static final String ADDITIONAL_CLASSPATH = "retz.classpath";
    private static final String DEFAULT_ADDITIONAL_CLASSPATH = "/opt/retz-server/lib";

    // Leeway seconds before old job entries get deleted by Retz.
    // As we may think of many race conditions on task finish at Mesos
    // and retry from Retz, updates, this value must be reasonably
    // large number, like several days that old tasks my not reincarnate.
    // For example, if a cluster usage accounting batch is scheduled
    // every week, this must be long enough that *all* jobs are
    // accounted and not deleted before any batch.
    private static final String GC = "retz.gc";
    private static final String GC_LEEWAY = "retz.gc.leeway";
    private static final int DEFAULT_GC_LEEWAY = 7 * 86400; // a week in seconds
    private static final String GC_INTERVAL = "retz.gc.interval";
    private static final int DEFAULT_GC_INTERVAL = 600; // 10 minutes in seconds


    public ServerConfiguration(InputStream in) throws IOException, URISyntaxException {
        super(in);


        Objects.requireNonNull(properties.getProperty(BIND_ADDRESS), "Host and port are required");
        Objects.requireNonNull(getMesosMaster(), "Mesos address must not be empty");

        uri = new URI(properties.getProperty(BIND_ADDRESS));
        // List<String> invalidBindAddresses = Arrays.asList(INVALID_BIND_ADDRESS);
        // loopback addresses must also be checked, but for now inttest uses localhost address
        if (uri.getHost().equals("0.0.0.0")) {
            LOG.error("retz.bind is told to Mesos; {}/32 should not be assigned", uri.getHost());
            throw new IllegalArgumentException();
        }
        if (uri.getPort() < 1024 || 65536 < uri.getPort()) {
            LOG.error("retz.bind must not use well known port, or just too large: {}", uri.getPort());
            throw new IllegalArgumentException();
        }

        maxSimultaneousJobs = Integer.parseInt(properties.getProperty(MAX_SIMULTANEOUS_JOBS, DEFAULT_MAX_SIMULTANEOUS_JOBS));
        if (maxSimultaneousJobs < 1) {
            throw new IllegalArgumentException(MAX_SIMULTANEOUS_JOBS + " must be positive");
        }

        databaseURL = properties.getProperty(DATABASE_URL, DEFAULT_DATABASE_URL);
        databaseDriver = properties.getProperty(DATABASE_DRIVER_CLASS, DEFAULT_DATABASE_DRIVER_CLASS);

        if ("root".equals(getUserName()) || getUserName().isEmpty()) {
            LOG.error("{} must not be 'root' nor empty", USER_NAME);
            throw new IllegalArgumentException("Invalid parameter: " + USER_NAME);
        }

        if (getRefuseSeconds() < 1) {
            throw new IllegalArgumentException(MESOS_REFUSE_SECONDS + " must be positive integer");
        }

        if (getJobQueueStrategy() == null) {
            throw new IllegalArgumentException(JOB_QUEUE_STRATEGY + " must be either fir or all");
        }

        LOG.info("Mesos master={}, principal={}, role={}, {}={}, {}={}, {}={}, {}={}, {}={}, {}={}, {}={}, {}={}, {}={}, {}={}",
                getMesosMaster(), getPrincipal(), getRole(), MAX_SIMULTANEOUS_JOBS, maxSimultaneousJobs,
                DATABASE_URL, databaseURL,
                MAX_STOCK_SIZE, getMaxStockSize(),
                USER_NAME, getUserName(),
                MESOS_REFUSE_SECONDS, getRefuseSeconds(),
                GC_LEEWAY, getGcLeeway(),
                GC_INTERVAL, getGcInterval(),
                MAX_LIST_JOB_SIZE, getMaxJobSize(),
                MAX_FILE_SIZE, getMaxFileSize(),
                JOB_QUEUE_STRATEGY, getJobQueueStrategy());
        LOG.info("{}={}", MESOS_FAILOVER_TIMEOUT, getFailoverTimeout());
    }

    public ServerConfiguration(String file) throws IOException, URISyntaxException {
        this(new FileInputStream(file));
    }

    // This is not only for Mesos master address, but mainly for ZooKeeper location
    // that enables Mesos master service discovery opaquely over failure.
    public String getMesosMaster() {
        return properties.getProperty(MESOS_LOC_KEY);
    }

    public URI getUri() {
        return uri;
{'fraction_non_alphanumeric': 0.09716487751169832, 'fraction_numerical': 0.012441508395265621, 'mean_word_length': 3.2236689142059984, 'pattern_counts': {'":': 4, '<': 15, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4047129', 'n_tokens_mistral': 5534, 'n_tokens_neox': 5263, 'n_words': 1547}