<commit_before>/*
 * The MIT License (MIT)
 *
 * Copyright (c) 2014 <EMAIL>
 *               2016 <NAME> <<EMAIL>>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

package com.dubture.jenkins.digitalocean;

import com.google.common.base.Strings;
import com.myjeeva.digitalocean.exception.DigitalOceanException;
import com.myjeeva.digitalocean.exception.RequestUnsuccessfulException;
import com.myjeeva.digitalocean.pojo.Droplet;
import com.myjeeva.digitalocean.pojo.Network;
import com.trilead.ssh2.Connection;
import com.trilead.ssh2.SCPClient;
import com.trilead.ssh2.Session;
import hudson.Util;
import hudson.model.TaskListener;
import hudson.remoting.Channel;
import hudson.slaves.SlaveComputer;
import hudson.util.TimeUnit2;
import jenkins.model.Jenkins;
import org.apache.commons.io.IOUtils;

import java.io.IOException;
import java.io.PrintStream;
import java.net.SocketTimeoutException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimeZone;

import static java.lang.String.format;

/**
 * The {@link ComputerLauncher} is responsible for:
 *
 * <ul>
 *   <li>Connecting to a slave via SSH</li>
 *   <li>Installing Java and the Jenkins agent to the slave</li>
 * </ul>
 *
 * @author <EMAIL>
 */
public class ComputerLauncher extends hudson.slaves.ComputerLauncher {
    /**
     * Connects to the given {@link Computer} via SSH and installs Java/Jenkins agent if necessary.
     */
    @Override
    public void launch(SlaveComputer _computer, TaskListener listener) {

        Computer computer = (Computer)_computer;
        PrintStream logger = listener.getLogger();

        Date startDate = new Date();
        logger.println("Start time: " + getUtcDate(startDate));

        final Connection conn;
        Connection cleanupConn = null;
        boolean successful = false;

        try {
            conn = connectToSsh(computer, logger);
            cleanupConn = conn;
            logger.println("Authenticating as " + computer.getRemoteAdmin());
            if (!conn.authenticateWithPublicKey(computer.getRemoteAdmin(), computer.getNode().getPrivateKey().toCharArray(), "")) {
                logger.println("Authentication failed");
                throw new Exception("Authentication failed");
            }

            final SCPClient scp = conn.createSCPClient();

            if (!runInitScript(computer, logger, conn, scp)) {
                return;
            }

            if (!installJava(logger, conn)) {
                return;
            }

            logger.println("Copying slave.jar");
            scp.put(Jenkins.getInstance().getJnlpJars("slave.jar").readFully(), "slave.jar","/tmp");
            String jvmOpts = Util.fixNull(computer.getNode().getJvmOpts());
            String launchString = "java " + jvmOpts + " -jar /tmp/slave.jar";
            logger.println("Launching slave agent: " + launchString);
            final Session sess = conn.openSession();
            sess.execCommand(launchString);
            computer.setChannel(sess.getStdout(), sess.getStdin(), logger, new Channel.Listener() {
                @Override
                public void onClosed(Channel channel, IOException cause) {
                    sess.close();
                    conn.close();
                }
            });

            successful = true;
        } catch (Exception e) {
            try {
                Jenkins.getInstance().removeNode(computer.getNode());
            } catch (Exception ee) {
                ee.printStackTrace(logger);
            }
            e.printStackTrace(logger);
        } finally {
            Date endDate = new Date();
            logger.println("Done setting up at: " + getUtcDate(endDate));
            logger.println("Done in " + TimeUnit2.MILLISECONDS.toSeconds(endDate.getTime() - startDate.getTime()) + " seconds");
            if(cleanupConn != null && !successful) {
                cleanupConn.close();
            }
        }
    }

    private boolean runInitScript(final Computer computer, final PrintStream logger, final Connection conn, final SCPClient scp)
            throws IOException, InterruptedException {

        String initScript = Util.fixEmptyAndTrim(computer.getNode().getInitScript());

        if (initScript == null) {
            return true;
        }
        if (conn.exec("test -e ~/.hudson-run-init", logger) == 0) {
            return true;
        }

        logger.println("Executing init script");
        scp.put(initScript.getBytes("UTF-8"), "init.sh", "/tmp", "0700");
        Session session = conn.openSession();
        session.requestDumbPTY(); // so that the remote side bundles stdout and stderr
        session.execCommand(buildUpCommand(computer, "/tmp/init.sh"));

        session.getStdin().close();    // nothing to write here
        session.getStderr().close();   // we are not supposed to get anything from stderr
        IOUtils.copy(session.getStdout(), logger);

        int exitStatus = waitCompletion(session);
        if (exitStatus != 0) {
            logger.println("init script failed: exit code=" + exitStatus);
            return false;
        }
        session.close();

        // Needs a tty to run sudo.
        session = conn.openSession();
        session.requestDumbPTY(); // so that the remote side bundles stdout and stderr
        session.execCommand(buildUpCommand(computer, "touch ~/.hudson-run-init"));
        session.close();

        return true;
    }

    private boolean installJava(final PrintStream logger, final Connection conn) throws IOException, InterruptedException {
        logger.println("Verifying that java exists");

        if (conn.exec("java -fullversion", logger) !=0) {
            logger.println("Installing Java");

            // TODO: Add support for non-debian based java installations
            // and let the user select the java version
            if (conn.exec("apt-get update -q && apt-get install -y openjdk-7-jdk", logger) !=0) {
                logger.println("Failed to download Java");
                return false;
            }
        }
        return true;
    }

    private Connection connectToSsh(Computer computer, PrintStream logger) throws RequestUnsuccessfulException, DigitalOceanException {

        // TODO: make configurable?
        final long timeout = TimeUnit2.MINUTES.toMillis(5);
        final long startTime = System.currentTimeMillis();
        final int sleepTime = 10;

        long waitTime;

        while ((waitTime = System.currentTimeMillis() - startTime) < timeout) {

            // Hack to fetch this each time through the loop to get the latest information.
            final Droplet droplet = DigitalOcean.getDroplet(
                    computer.getCloud().getAuthToken(),
                    computer.getNode().getDropletId());

            if (isDropletStarting(droplet)) {<commit_msg>Basic non-debian java installer compatibility. (#21)<commit_after>/*
 * The MIT License (MIT)
 *
 * Copyright (c) 2014 <EMAIL>
 *               2016 <NAME> <<EMAIL>>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

package com.dubture.jenkins.digitalocean;

import com.google.common.base.Strings;
import com.myjeeva.digitalocean.exception.DigitalOceanException;
import com.myjeeva.digitalocean.exception.RequestUnsuccessfulException;
import com.myjeeva.digitalocean.pojo.Droplet;
import com.myjeeva.digitalocean.pojo.Network;
import com.trilead.ssh2.Connection;
import com.trilead.ssh2.SCPClient;
import com.trilead.ssh2.Session;
import hudson.Util;
import hudson.model.TaskListener;
import hudson.remoting.Channel;
import hudson.slaves.SlaveComputer;
import hudson.util.TimeUnit2;
import jenkins.model.Jenkins;
import org.apache.commons.io.IOUtils;

import java.io.IOException;
import java.io.PrintStream;
import java.net.SocketTimeoutException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimeZone;

import static java.lang.String.format;

/**
 * The {@link ComputerLauncher} is responsible for:
 *
 * <ul>
 *   <li>Connecting to a slave via SSH</li>
 *   <li>Installing Java and the Jenkins agent to the slave</li>
 * </ul>
 *
 * @author <EMAIL>
 */
public class ComputerLauncher extends hudson.slaves.ComputerLauncher {
    /**
     * Wrapper around package installers so that the provision process can install extra packages as needed.
     */
    private enum PackageInstaller {

        APT_GET("apt-get") {
            @Override
            public int installPackage(String packageToInstall, PrintStream logger, Connection conn) throws IOException, InterruptedException {
                return conn.exec("apt-get update -q && apt-get install -y " + packageToInstall, logger);
            }
        }, YUM("yum") {
            @Override
            public int installPackage(String packageToInstall, PrintStream logger, Connection conn) throws IOException, InterruptedException {
                return conn.exec("yum install -y " + packageToInstall, logger);
            }
        };

        private String commandName;

        PackageInstaller(String commandName) {
            this.commandName = commandName;
        }

        public boolean isPresent(final PrintStream logger, final Connection conn) throws IOException, InterruptedException {
            return conn.exec("which " + this.commandName, logger) == 0;
        }

        public abstract int installPackage(final String packageToInstall, final PrintStream logger, final Connection conn) throws IOException, InterruptedException;

        public String getCommandName() {
            return commandName;
        }
    }

    /**
     * Connects to the given {@link Computer} via SSH and installs Java/Jenkins agent if necessary.
     */
    @Override
    public void launch(SlaveComputer _computer, TaskListener listener) {

        Computer computer = (Computer)_computer;
        PrintStream logger = listener.getLogger();

        Date startDate = new Date();
        logger.println("Start time: " + getUtcDate(startDate));

        final Connection conn;
        Connection cleanupConn = null;
        boolean successful = false;

        try {
            conn = connectToSsh(computer, logger);
            cleanupConn = conn;
            logger.println("Authenticating as " + computer.getRemoteAdmin());
            if (!conn.authenticateWithPublicKey(computer.getRemoteAdmin(), computer.getNode().getPrivateKey().toCharArray(), "")) {
                logger.println("Authentication failed");
                throw new Exception("Authentication failed");
            }

            final SCPClient scp = conn.createSCPClient();

            if (!runInitScript(computer, logger, conn, scp)) {
                return;
            }

            if (!installJava(logger, conn)) {
                return;
            }

            logger.println("Copying slave.jar");
            scp.put(Jenkins.getInstance().getJnlpJars("slave.jar").readFully(), "slave.jar","/tmp");
            String jvmOpts = Util.fixNull(computer.getNode().getJvmOpts());
            String launchString = "java " + jvmOpts + " -jar /tmp/slave.jar";
            logger.println("Launching slave agent: " + launchString);
            final Session sess = conn.openSession();
            sess.execCommand(launchString);
            computer.setChannel(sess.getStdout(), sess.getStdin(), logger, new Channel.Listener() {
                @Override
                public void onClosed(Channel channel, IOException cause) {
                    sess.close();
                    conn.close();
                }
            });

            successful = true;
        } catch (Exception e) {
            try {
                Jenkins.getInstance().removeNode(computer.getNode());
            } catch (Exception ee) {
                ee.printStackTrace(logger);
            }
            e.printStackTrace(logger);
        } finally {
            Date endDate = new Date();
            logger.println("Done setting up at: " + getUtcDate(endDate));
            logger.println("Done in " + TimeUnit2.MILLISECONDS.toSeconds(endDate.getTime() - startDate.getTime()) + " seconds");
            if(cleanupConn != null && !successful) {
                cleanupConn.close();
            }
        }
    }

    private boolean runInitScript(final Computer computer, final PrintStream logger, final Connection conn, final SCPClient scp)
            throws IOException, InterruptedException {

        String initScript = Util.fixEmptyAndTrim(computer.getNode().getInitScript());

        if (initScript == null) {
            return true;
        }
        if (conn.exec("test -e ~/.hudson-run-init", logger) == 0) {
            return true;
        }

        logger.println("Executing init script");
        scp.put(initScript.getBytes("UTF-8"), "init.sh", "/tmp", "0700");
        Session session = conn.openSession();
        session.requestDumbPTY(); // so that the remote side bundles stdout and stderr
        session.execCommand(buildUpCommand(computer, "/tmp/init.sh"));

        session.getStdin().close();    // nothing to write here
        session.getStderr().close();   // we are not supposed to get anything from stderr
        IOUtils.copy(session.getStdout(), logger);

        int exitStatus = waitCompletion(session);
        if (exitStatus != 0) {
            logger.println("init script failed: exit code=" + exitStatus);
            return false;
        }
        session.close();

        // Needs a tty to run sudo.
        session = conn.openSession();
        session.requestDumbPTY(); // so that the remote side bundles stdout and stderr
        session.execCommand(buildUpCommand(computer, "touch ~/.hudson-run-init"));
        session.close();

        return true;
    }

    private boolean installJava(final PrintStream logger, final Connection conn) throws IOException, InterruptedException {
        logger.println("Verifying that java exists");

        if (conn.exec("java -fullversion", logger) !=0) {
            logger.println("Installing Java");

            //TODO Web UI to let users install a custom java (or any other type of tool) package.
            for (PackageInstaller installer : PackageInstaller.values()) {
                logger.println("Testing for " + installer.getCommandName() + " package installer!");
                if(installer.isPresent(logger, conn)) {
                    logger.println("Found "+installer.getCommandName()+"! Attempting java install...");
                    if(installer.installPackage("openjdk-7-jdk", logger, conn) != 0) {
                        logger.println("Could not install java using "+installer.getCommandName()+"! Attempting the rest of the installers...");
                    } else {
                        return true;
                    }
                }
            }

            logger.println("Java could not be installed using any of the supported package installer!");
            return false;
        }
        return true;
    }

    private Connection connectToSsh(Computer computer, PrintStream logger) throws RequestUnsuccessfulException, DigitalOceanException {

        // TODO: make configurable?
        final long timeout = TimeUnit2.MINUTES.toMillis(5);
        final long startTime = System.currentTimeMillis();
        final int sleepTime = 10;

        long waitTime;

        while ((waitTime = System.currentTimeMillis() - startTime) < timeout) {

            // Hack to fetch this each time through the loop to get the latest information.
            final Droplet droplet = DigitalOcean.getDroplet(
                    computer.getCloud().getAuthToken(),
                    computer.getNode().getDropletId());

            if (isDropletStarting(droplet)) {
{'fraction_non_alphanumeric': 0.10020265705922089, 'fraction_numerical': 0.003208736770997523, 'mean_word_length': 2.5185185185185186, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3523583', 'n_tokens_mistral': 4744, 'n_tokens_neox': 4499, 'n_words': 1367}