<commit_before>package com.hazelcast.stabilizer.provisioner

import com.google.common.base.Predicate
import com.hazelcast.logging.ILogger
import com.hazelcast.logging.Logger
import com.hazelcast.stabilizer.Utils
import com.hazelcast.stabilizer.agent.AgentRemoteService
import com.hazelcast.stabilizer.agent.workerjvm.WorkerJvmManager
import org.jclouds.ContextBuilder
import org.jclouds.compute.ComputeService
import org.jclouds.compute.ComputeServiceContext
import org.jclouds.compute.domain.NodeMetadata
import org.jclouds.compute.domain.Template
import org.jclouds.compute.domain.TemplateBuilderSpec
import org.jclouds.logging.log4j.config.Log4JLoggingModule
import org.jclouds.scriptbuilder.statements.login.AdminAccess
import org.jclouds.sshj.config.SshjSshClientModule

import java.util.concurrent.*

import static com.hazelcast.stabilizer.Utils.*
import static java.lang.String.format
import static java.util.Arrays.asList
import static org.jclouds.compute.config.ComputeServiceProperties.POLL_INITIAL_PERIOD
import static org.jclouds.compute.config.ComputeServiceProperties.POLL_MAX_PERIOD

//https://jclouds.apache.org/start/compute/ good read
//https://github.com/jclouds/jclouds-examples/blob/master/compute-basics/src/main/java/org/jclouds/examples/compute/basics/MainApp.java
//https://github.com/jclouds/jclouds-examples/blob/master/minecraft-compute/src/main/java/org/jclouds/examples/minecraft/NodeManager.java
public class Provisioner {
    private final static ILogger log = Logger.getLogger(Provisioner.class.getName());

    def config
    final String STABILIZER_HOME = Utils.getStablizerHome().getAbsolutePath()
    final File CONF_DIR = new File(STABILIZER_HOME, "conf");
    final File agentsFile = new File("agents.txt")
    //big number of threads, but they are used to offload ssh tasks. So there is no load on this machine..
    final ExecutorService executor = Executors.newFixedThreadPool(10);

    final List<String> privateIps = Collections.synchronizedList(new LinkedList<String>());

    Provisioner() {
        log.info("Hazelcast Stabilizer Provisioner");
        log.info(format("Version: %s", getVersion()));
        log.info(format("STABILIZER_HOME: %s", STABILIZER_HOME));

        Properties props = new Properties()

        new File("stabilizer.properties").withInputStream {
            stream -> props.load(stream)
        }
        config = new ConfigSlurper().parse(props)

        if (!agentsFile.exists()) {
            agentsFile.createNewFile()
        }
        agentsFile.text.eachLine { String line -> privateIps << line }
    }

    void installAgent(String ip) {
//        echoImportant "Installing Agent on ${ip}"

//        echo "Copying stabilizer files"
        //first we remove the old lib files to prevent different versions of the same jar to bite us.
        sshQuiet ip, "rm -fr hazelcast-stabilizer-${getVersion()}/lib"
        //then we copy the stabilizer directory
        scpToRemote ip, STABILIZER_HOME, ""

//        echoImportant("Successfully installed Agent on ${ip}");
    }


    void startAgents() {
        echoImportant("Starting ${privateIps.size()} Agents");

        for (String ip : privateIps) {
            echo "Killing Agent $ip"
            ssh ip, "killall -9 java || true"
        }

        for (String ip : privateIps) {
            echo "Starting Agent $ip"
            ssh ip, "nohup hazelcast-stabilizer-${getVersion()}/bin/agent > agent.out 2> agent.err < /dev/null &"
        }

        echoImportant("Successfully started ${privateIps.size()} Agents");
    }

    void startAgent(String ip) {
//        echoImportant("Starting ${privateIps.size()} Agents");

//        for (String ip : privateIps) {
//            echo "Killing Agent $ip"
        ssh ip, "killall -9 java || true"
//        }

        //       for (String ip : privateIps) {
        //           echo "Starting Agent $ip"
        ssh ip, "nohup hazelcast-stabilizer-${getVersion()}/bin/agent > agent.out 2> agent.err < /dev/null &"
        //       }

        //       echoImportant("Successfully started ${privateIps.size()} Agents");
    }

    void killAgents() {
        echoImportant("Killing ${privateIps.size()} Agents");

        for (String ip : privateIps) {
            echo "Killing Agent $ip"
            ssh ip, "killall -9 java || true"
        }

        echoImportant("Successfully killed ${privateIps.size()} Agents");
    }

    void installAgents() {
        privateIps.each { String ip -> installAgent(ip) }
    }

    int calcClusterSize(String sizeType) {
        try {
            return Integer.parseInt(sizeType)
        } catch (NumberFormatException e) {
            println "Unknown size: $sizeType"
            System.exit 1
        }
    }

    void scale(String sizeType) {
        int delta = calcClusterSize(sizeType) - privateIps.size();
        if (delta == 0) {
            echo("Ignoring spawn machines, desired number of machines already exists");
        } else if (delta < 0) {
            terminate(-delta);
        } else {
            scaleUp(delta);
        }
    }

    private int[] inboundPorts() {
        List<Integer> ports = new ArrayList<Integer>();
        ports.add(22);
        //todo:the following 2 ports should not be needed
        ports.add(443);
        ports.add(80);
        ports.add(AgentRemoteService.PORT);
        ports.add(WorkerJvmManager.PORT);
        for (int k = 5701; k < 5901; k++) {
            ports.add(k);
        }

        int[] result = new int[ports.size()];
        for (int k = 0; k < result.length; k++) {
            result[k] = ports[k];
        }
        return result;
    }

    private void scaleUp(int delta) {
        echoImportant("Provisioning ${delta} ${config.CLOUD_PROVIDER} machines");
        echo(config.MACHINE_SPEC);

        long startTimeMs = System.currentTimeMillis();

        if ("outofthebox".equals(config.JDK_FLAVOR)) {
            log.info("Machines will use Java: Out of the Box.");
        } else {
            log.info("Machines will use Java: ${config.JDK_FLAVOR} ${config.JDK_VERSION}")
        }

        ComputeService compute = getComputeService()

        echo("Created compute")

        Template template = compute.templateBuilder()
                .from(TemplateBuilderSpec.parse(config.MACHINE_SPEC))
                .build();

        echo("Created template")

        template.getOptions()
                .inboundPorts(inboundPorts())
                .runScript(AdminAccess.standard())
                .securityGroups(config.SECURITY_GROUP)

        echo("Creating nodes")

        Set<Future> futures = new LinkedList<Future>();
        echo("Created machines, waiting for startup (can take a few minutes)")
        echo("Batches: " + calcBatches(delta))


        for (int batch : calcBatches(delta)) {

            Set<NodeMetadata> nodes = compute.createNodesInGroup("stabilizer-agent", batch, template)

            for (NodeMetadata node : nodes) {
                String ip = node.privateAddresses.iterator().next()
                echo("\t" + ip + " LAUNCHED");
                appendText(ip + "\n", agentsFile)
                privateIps.add(ip)
            }

            for (NodeMetadata node : nodes) {
                Future f = executor.submit(new InstallNodeTask(node, compute));
                futures.add(f);
            }

            sleepSeconds(30)
        }

        for (Future f : futures) {
            try {
                f.get();
            } catch (ExecutionException e) {
                log.severe("Failed provision", e)
                System.exit(1)
            }
        }

        long durationMs = System.currentTimeMillis() - startTimeMs;
        echo("Duration: " + secondsToHuman(TimeUnit.MILLISECONDS.toSeconds(durationMs)))
        echoImportant("Successfully provisioned ${delta} ${config.CLOUD_PROVIDER} machines ");
    }

    private class InstallNodeTask implements Runnable {
        private final NodeMetadata node;
        private final ComputeService compute;
        private final String ip;

        InstallNodeTask(NodeMetadata node, ComputeService compute) {
            this.node = node
            this.compute = compute
            this.ip = node.getPrivateAddresses().iterator().next();
        }

        public void run() {
            //install java if needed
            if (!"outofthebox".equals(config.JDK_FLAVOR)) {
                ssh(ip, "touch install-java.sh")
                ssh(ip, "chmod +x install-java.sh")
                scpToRemote(ip, getJavaInstallScript().getAbsolutePath(), "install-java.sh")
                ssh(ip, "sudo bash install-java.sh")
                echo("\t" + ip + " JAVA INSTALLED");
            }

            installAgent(ip)
            echo("\t" + ip + " STABILIZER AGENT INSTALLED");

            startAgent(ip)
            echo("\t" + ip + " STABILIZER AGENT STARTED");
        }

    }

    private int[] calcBatches(int size) {
        List<Integer> batches = new LinkedList<Integer>();
        int batchSize = Integer.parseInt(config.CLOUD_BATCH_SIZE);
        while (size > 0) {
            int x = size >= batchSize ? batchSize : size;
            batches.add(x);
            size -= x;
        }

        int[] result = new int[batches.size()];
        for (int k = 0; k < result.length; k++) {
            result[k] = batches.get(k);
        }
        return result;
    }

    private ComputeService getComputeService() {
        //http://javadocs.jclouds.cloudbees.net/org/jclouds/compute/config/ComputeServiceProperties.html
        Properties overrides = new Properties();
        overrides.setProperty(POLL_INITIAL_PERIOD, config.CLOUD_POLL_INITIAL_PERIOD);
        overrides.setProperty(POLL_MAX_PERIOD, config.CLOUD_POLL_MAX_PERIOD);

        return ContextBuilder.newBuilder(config.CLOUD_PROVIDER)
                .overrides(overrides)
                .credentials(config.CLOUD_IDENTITY, config.CLOUD_CREDENTIAL)
                .modules(asList(new Log4JLoggingModule(), new SshjSshClientModule()))
                .buildView(ComputeServiceContext.class)
                .getComputeService();
    }<commit_msg>Trying to load google key<commit_after>package com.hazelcast.stabilizer.provisioner

import com.google.common.base.Predicate
import com.hazelcast.logging.ILogger
import com.hazelcast.logging.Logger
import com.hazelcast.stabilizer.Utils
import com.hazelcast.stabilizer.agent.AgentRemoteService
import com.hazelcast.stabilizer.agent.workerjvm.WorkerJvmManager
import org.jclouds.ContextBuilder
import org.jclouds.compute.ComputeService
import org.jclouds.compute.ComputeServiceContext
import org.jclouds.compute.domain.NodeMetadata
import org.jclouds.compute.domain.Template
import org.jclouds.compute.domain.TemplateBuilderSpec
import org.jclouds.logging.log4j.config.Log4JLoggingModule
import org.jclouds.scriptbuilder.statements.login.AdminAccess
import org.jclouds.sshj.config.SshjSshClientModule

import java.util.concurrent.*

import static com.hazelcast.stabilizer.Utils.*
import static java.lang.String.format
import static java.util.Arrays.asList
import static org.jclouds.compute.config.ComputeServiceProperties.POLL_INITIAL_PERIOD
import static org.jclouds.compute.config.ComputeServiceProperties.POLL_MAX_PERIOD

//https://jclouds.apache.org/start/compute/ good read
//https://github.com/jclouds/jclouds-examples/blob/master/compute-basics/src/main/java/org/jclouds/examples/compute/basics/MainApp.java
//https://github.com/jclouds/jclouds-examples/blob/master/minecraft-compute/src/main/java/org/jclouds/examples/minecraft/NodeManager.java
public class Provisioner {
    private final static ILogger log = Logger.getLogger(Provisioner.class.getName());

    def config
    final String STABILIZER_HOME = Utils.getStablizerHome().getAbsolutePath()
    final File CONF_DIR = new File(STABILIZER_HOME, "conf");
    final File agentsFile = new File("agents.txt")
    //big number of threads, but they are used to offload ssh tasks. So there is no load on this machine..
    final ExecutorService executor = Executors.newFixedThreadPool(10);

    final List<String> privateIps = Collections.synchronizedList(new LinkedList<String>());

    Provisioner() {
        log.info("Hazelcast Stabilizer Provisioner");
        log.info(format("Version: %s", getVersion()));
        log.info(format("STABILIZER_HOME: %s", STABILIZER_HOME));

        Properties props = new Properties()

        new File("stabilizer.properties").withInputStream {
            stream -> props.load(stream)
        }
        config = new ConfigSlurper().parse(props)

        if (!agentsFile.exists()) {
            agentsFile.createNewFile()
        }
        agentsFile.text.eachLine { String line -> privateIps << line }
    }

    void installAgent(String ip) {
//        echoImportant "Installing Agent on ${ip}"

//        echo "Copying stabilizer files"
        //first we remove the old lib files to prevent different versions of the same jar to bite us.
        sshQuiet ip, "rm -fr hazelcast-stabilizer-${getVersion()}/lib"
        //then we copy the stabilizer directory
        scpToRemote ip, STABILIZER_HOME, ""

//        echoImportant("Successfully installed Agent on ${ip}");
    }


    void startAgents() {
        echoImportant("Starting ${privateIps.size()} Agents");

        for (String ip : privateIps) {
            echo "Killing Agent $ip"
            ssh ip, "killall -9 java || true"
        }

        for (String ip : privateIps) {
            echo "Starting Agent $ip"
            ssh ip, "nohup hazelcast-stabilizer-${getVersion()}/bin/agent > agent.out 2> agent.err < /dev/null &"
        }

        echoImportant("Successfully started ${privateIps.size()} Agents");
    }

    void startAgent(String ip) {
//        echoImportant("Starting ${privateIps.size()} Agents");

//        for (String ip : privateIps) {
//            echo "Killing Agent $ip"
        ssh ip, "killall -9 java || true"
//        }

        //       for (String ip : privateIps) {
        //           echo "Starting Agent $ip"
        ssh ip, "nohup hazelcast-stabilizer-${getVersion()}/bin/agent > agent.out 2> agent.err < /dev/null &"
        //       }

        //       echoImportant("Successfully started ${privateIps.size()} Agents");
    }

    void killAgents() {
        echoImportant("Killing ${privateIps.size()} Agents");

        for (String ip : privateIps) {
            echo "Killing Agent $ip"
            ssh ip, "killall -9 java || true"
        }

        echoImportant("Successfully killed ${privateIps.size()} Agents");
    }

    void installAgents() {
        privateIps.each { String ip -> installAgent(ip) }
    }

    int calcClusterSize(String sizeType) {
        try {
            return Integer.parseInt(sizeType)
        } catch (NumberFormatException e) {
            println "Unknown size: $sizeType"
            System.exit 1
        }
    }

    void scale(String sizeType) {
        int delta = calcClusterSize(sizeType) - privateIps.size();
        if (delta == 0) {
            echo("Ignoring spawn machines, desired number of machines already exists");
        } else if (delta < 0) {
            terminate(-delta);
        } else {
            scaleUp(delta);
        }
    }

    private int[] inboundPorts() {
        List<Integer> ports = new ArrayList<Integer>();
        ports.add(22);
        //todo:the following 2 ports should not be needed
        ports.add(443);
        ports.add(80);
        ports.add(AgentRemoteService.PORT);
        ports.add(WorkerJvmManager.PORT);
        for (int k = 5701; k < 5901; k++) {
            ports.add(k);
        }

        int[] result = new int[ports.size()];
        for (int k = 0; k < result.length; k++) {
            result[k] = ports[k];
        }
        return result;
    }

    private void scaleUp(int delta) {
        echoImportant("Provisioning ${delta} ${config.CLOUD_PROVIDER} machines");
        echo(config.MACHINE_SPEC);

        long startTimeMs = System.currentTimeMillis();

        if ("outofthebox".equals(config.JDK_FLAVOR)) {
            log.info("Machines will use Java: Out of the Box.");
        } else {
            log.info("Machines will use Java: ${config.JDK_FLAVOR} ${config.JDK_VERSION}")
        }

        ComputeService compute = getComputeService()

        echo("Created compute")

        Template template = compute.templateBuilder()
                .from(TemplateBuilderSpec.parse(config.MACHINE_SPEC))
                .build();

        echo("Created template")

        template.getOptions()
                .inboundPorts(inboundPorts())
                .runScript(AdminAccess.standard())
                .securityGroups(config.SECURITY_GROUP)

        echo("Creating nodes")

        Set<Future> futures = new LinkedList<Future>();
        echo("Created machines, waiting for startup (can take a few minutes)")
        echo("Batches: " + calcBatches(delta))


        for (int batch : calcBatches(delta)) {

            Set<NodeMetadata> nodes = compute.createNodesInGroup("stabilizer-agent", batch, template)

            for (NodeMetadata node : nodes) {
                String ip = node.privateAddresses.iterator().next()
                echo("\t" + ip + " LAUNCHED");
                appendText(ip + "\n", agentsFile)
                privateIps.add(ip)
            }

            for (NodeMetadata node : nodes) {
                Future f = executor.submit(new InstallNodeTask(node, compute));
                futures.add(f);
            }

            sleepSeconds(30)
        }

        for (Future f : futures) {
            try {
                f.get();
            } catch (ExecutionException e) {
                log.severe("Failed provision", e)
                System.exit(1)
            }
        }

        long durationMs = System.currentTimeMillis() - startTimeMs;
        echo("Duration: " + secondsToHuman(TimeUnit.MILLISECONDS.toSeconds(durationMs)))
        echoImportant("Successfully provisioned ${delta} ${config.CLOUD_PROVIDER} machines ");
    }

    private class InstallNodeTask implements Runnable {
        private final NodeMetadata node;
        private final ComputeService compute;
        private final String ip;

        InstallNodeTask(NodeMetadata node, ComputeService compute) {
            this.node = node
            this.compute = compute
            this.ip = node.getPrivateAddresses().iterator().next();
        }

        public void run() {
            //install java if needed
            if (!"outofthebox".equals(config.JDK_FLAVOR)) {
                ssh(ip, "touch install-java.sh")
                ssh(ip, "chmod +x install-java.sh")
                scpToRemote(ip, getJavaInstallScript().getAbsolutePath(), "install-java.sh")
                ssh(ip, "sudo bash install-java.sh")
                echo("\t" + ip + " JAVA INSTALLED");
            }

            installAgent(ip)
            echo("\t" + ip + " STABILIZER AGENT INSTALLED");

            startAgent(ip)
            echo("\t" + ip + " STABILIZER AGENT STARTED");
        }

    }

    private int[] calcBatches(int size) {
        List<Integer> batches = new LinkedList<Integer>();
        int batchSize = Integer.parseInt(config.CLOUD_BATCH_SIZE);
        while (size > 0) {
            int x = size >= batchSize ? batchSize : size;
            batches.add(x);
            size -= x;
        }

        int[] result = new int[batches.size()];
        for (int k = 0; k < result.length; k++) {
            result[k] = batches.get(k);
        }
        return result;
    }

    private ComputeService getComputeService() {
        //http://javadocs.jclouds.cloudbees.net/org/jclouds/compute/config/ComputeServiceProperties.html
        Properties overrides = new Properties();
        overrides.setProperty(POLL_INITIAL_PERIOD, config.CLOUD_POLL_INITIAL_PERIOD);
        overrides.setProperty(POLL_MAX_PERIOD, config.CLOUD_POLL_MAX_PERIOD);

        String credentials = config.CLOUD_CREDENTIALS;
        File file = new File(credentials);
        if(file.exists()){
            credentials = Utils.fileAsText(file);
        }

        return ContextBuilder.newBuilder(config.CLOUD_PROVIDER)
                .overrides(overrides)
                .credentials(config.CLOUD_IDENTITY, credentials)
                .modules(asList(new Log4JLoggingModule(), new SshjSshClientModule()))
                .buildView(ComputeServiceContext.class)
                .getComputeService();
    }
{'fraction_non_alphanumeric': 0.1134585289514867, 'fraction_numerical': 0.003423317683881064, 'mean_word_length': 2.3272046859746176, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 39, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2565020', 'n_tokens_mistral': 6121, 'n_tokens_neox': 5887, 'n_words': 1268}