<commit_before>package com.miniplay.minicortex.modules.worker.drivers.docker;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.miniplay.common.CommandExecutor;
import com.miniplay.common.Debugger;
import com.miniplay.common.Stats;
import com.miniplay.minicortex.config.Config;
import com.miniplay.minicortex.config.ConfigManager;
import com.miniplay.minicortex.exceptions.InvalidProvisionParams;
import com.miniplay.minicortex.modules.balancer.ElasticBalancer;
import com.miniplay.minicortex.modules.worker.drivers.AbstractWorkerDriver;
import com.miniplay.minicortex.modules.worker.drivers.dockermachine.ProvisionThread;
import com.miniplay.minicortex.modules.worker.exceptions.DriverException;
import com.spotify.docker.client.*;
import com.spotify.docker.client.exceptions.DockerException;
import com.spotify.docker.client.messages.*;
import com.sun.org.apache.xml.internal.resolver.helpers.Debug;
import com.sun.org.apache.xml.internal.security.encryption.ReferenceList;

import java.io.IOException;
import java.math.BigInteger;
import java.net.URI;
import java.nio.file.Paths;
import java.security.SecureRandom;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;

/**
 * Docker manager Object
 * Created by ret on 4/12/15.
 */
public class DockerDriver extends AbstractWorkerDriver {

    private DockerClient dockerClient = null;
    protected ElasticBalancer elasticBalancer = null;
    public Boolean isLoaded = false;
    public volatile ConcurrentHashMap<String, Worker> workers = new ConcurrentHashMap<String, Worker>();

    private Config config = null;
    private ArrayList<String> dockerEnvironmentVars;
    private String dockerEnvironment = null;
    private String dockerServiceName = null;

    /**
     * DockerMachineDriver constructor
     * @param elasticBalancer ElasticBalancer
     */
    public DockerDriver(ElasticBalancer elasticBalancer) {
        super(elasticBalancer);
        this.elasticBalancer = elasticBalancer;


        // Load config & Instance new docker client
        try {

            this.loadConfig();

            if(config.DOCKER_CONNECTION.get("HOST") == null) {
                throw new DriverException("Docker connection host not defined");
            }
            if(config.DOCKER_CONNECTION.get("HOST") != null && config.DOCKER_CONNECTION.get("HOST").contains("https://") && config.DOCKER_CONNECTION.get("CERTS_DIR") == null) {
                throw new DriverException("Docker connection host defined but it's certs dir NOT");
            }
            if(config.DOCKER_CONNECTION.get("HOST").contains("unix:///")) { // Use the unix socket way when available
                this.dockerClient = new DefaultDockerClient(config.DOCKER_CONNECTION.get("HOST"));
            } else { // Use https connection way
                this.dockerClient = DefaultDockerClient.builder()
                        .uri(URI.create(config.DOCKER_CONNECTION.get("HOST")))
                        .dockerCertificates(new DockerCertificates(Paths.get(config.DOCKER_CONNECTION.get("CERTS_DIR"))))
                        .build();
            }

        } catch(Exception e) {
            System.out.println("Docker client instance exception: " + e.getMessage());
        }

        Debugger.getInstance().print("DockerDriver Loaded OK",this.getClass());
    }

    /**
     * Load conf into obj
     */
    private void loadConfig() throws DriverException {
        this.config = ConfigManager.getConfig();

        this.setMinWorkers(config.DOCKER_MIN_CONTAINERS);
        this.setMaxWorkers(config.DOCKER_MAX_CONTAINERS);
        this.setMaxBootsInLoop(config.DOCKER_MAX_BOOTS_IN_LOOP);
        this.setMaxShutdownsInLoop(config.DOCKER_MAX_SHUTDOWNS_IN_LOOP);
        this.setDockerEnvironment(config.DOCKER_ENV_VARS);
        this.setDockerServiceName(config.DOCKER_SERVICE_NAME);

        this.isLoaded = true;
    }

    /**
     * Get registered worker by worker name
     * @param name String
     * @return Worker
     */
    public Worker getWorker(String name) {
        return this.workers.get(name);
    }

    /**
     * Load workers from "docker-machine ls" into application
     */
    public void loadWorkers() {
        try {

            // @TODO: Load workers
            dockerClient.listContainers(DockerClient.ListContainersParam.withLabel("environment",this.dockerEnvironment));



        } catch (Exception e) {
            System.out.println(Debugger.PREPEND_OUTPUT_DOCKER + "EXCEPTION: " + e.getMessage());
            e.printStackTrace();
        }


    }

    /**
     * Registers a new worker into the application
     * @return Boolean
     */
    public Boolean registerWorker(String id, String name, String image, ArrayList<String> bindPorts, ArrayList<String> envVars, ArrayList<String> mountedVolumes, ContainerState state) {
        try {
            Worker worker = new Worker(this, id, this.dockerClient, name, image, bindPorts, envVars, mountedVolumes, state);
            Boolean workerExists = this.workers.get(name) != null;
            if(workerExists) {
                this.workers.replace(name, worker);
            } else {
                this.workers.put(name, worker);
            }

            return true;
        } catch (Exception e) {
            System.out.println(Debugger.PREPEND_OUTPUT_DOCKER + e.getMessage());
            return false;
        }
    }

    /**
     * Provision a new worker into the Docker cluster
     * @param workerName String
     */
    public void provisionWorker(String workerName) {
        try {
            Debugger.getInstance().printOutput("Provisioning new worker #"+workerName);

            //Provision new worker (docker container run?)


            final HostConfig.Builder hostConfigBuilder = HostConfig.builder();

            // Set exposed ports
//            for (String volume:this.getConfig().DOCKER_PORTS) {
//                hostConfigBuilder.portb(volume);
//            }


            // Set volumes
            for (String volume:this.getConfig().DOCKER_VOLUMES) {
                hostConfigBuilder.appendBinds(volume);
            }


            ContainerConfig.Builder containerBuilder = ContainerConfig.builder()
                .image(this.getConfig().DOCKER_IMAGE)
                .env(config.DOCKER_ENV_VARS)
                .labels(ImmutableMap.of("environment",this.dockerEnvironment,"service",this.dockerServiceName))
                .hostConfig(hostConfigBuilder.build())
                .hostname(workerName);


            final ContainerCreation container = this.dockerClient.createContainer(containerBuilder.build(),workerName);

            ContainerInfo containerInfo = dockerClient.inspectContainer(container.id());<commit_msg> Refactor whole service to allow different balance drivers for workers. i.e: balance workers managed by docker-machine for AWS instances or balance workers managed by docker directly on the machine.<commit_after>package com.miniplay.minicortex.modules.worker.drivers.docker;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.miniplay.common.CommandExecutor;
import com.miniplay.common.Debugger;
import com.miniplay.common.Stats;
import com.miniplay.minicortex.config.Config;
import com.miniplay.minicortex.config.ConfigManager;
import com.miniplay.minicortex.exceptions.InvalidProvisionParams;
import com.miniplay.minicortex.modules.balancer.ElasticBalancer;
import com.miniplay.minicortex.modules.worker.drivers.AbstractWorkerDriver;
import com.miniplay.minicortex.modules.worker.drivers.dockermachine.ProvisionThread;
import com.miniplay.minicortex.modules.worker.exceptions.DriverException;
import com.spotify.docker.client.*;
import com.spotify.docker.client.exceptions.DockerException;
import com.spotify.docker.client.messages.*;
import com.sun.org.apache.xml.internal.resolver.helpers.Debug;
import com.sun.org.apache.xml.internal.security.encryption.ReferenceList;

import java.io.IOException;
import java.math.BigInteger;
import java.net.URI;
import java.nio.file.Paths;
import java.security.SecureRandom;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;

/**
 * Docker manager Object
 * Created by ret on 4/12/15.
 */
public class DockerDriver extends AbstractWorkerDriver {

    private DockerClient dockerClient = null;
    protected ElasticBalancer elasticBalancer = null;
    public Boolean isLoaded = false;
    public volatile ConcurrentHashMap<String, Worker> workers = new ConcurrentHashMap<String, Worker>();

    private Config config = null;
    private ArrayList<String> dockerEnvironmentVars;
    private String dockerEnvironment = null;
    private String dockerServiceName = null;

    /**
     * DockerMachineDriver constructor
     * @param elasticBalancer ElasticBalancer
     */
    public DockerDriver(ElasticBalancer elasticBalancer) {
        super(elasticBalancer);
        this.elasticBalancer = elasticBalancer;


        // Load config & Instance new docker client
        try {

            this.loadConfig();

            if(config.DOCKER_CONNECTION.get("HOST") == null) {
                throw new DriverException("Docker connection host not defined");
            }
            if(config.DOCKER_CONNECTION.get("HOST") != null && config.DOCKER_CONNECTION.get("HOST").contains("https://") && config.DOCKER_CONNECTION.get("CERTS_DIR") == null) {
                throw new DriverException("Docker connection host defined but it's certs dir NOT");
            }
            if(config.DOCKER_CONNECTION.get("HOST").contains("unix:///")) { // Use the unix socket way when available
                this.dockerClient = new DefaultDockerClient(config.DOCKER_CONNECTION.get("HOST"));
            } else { // Use https connection way
                this.dockerClient = DefaultDockerClient.builder()
                        .uri(URI.create(config.DOCKER_CONNECTION.get("HOST")))
                        .dockerCertificates(new DockerCertificates(Paths.get(config.DOCKER_CONNECTION.get("CERTS_DIR"))))
                        .build();
            }

        } catch(Exception e) {
            System.out.println("Docker client instance exception: " + e.getMessage());
        }

        Debugger.getInstance().print("DockerDriver Loaded OK",this.getClass());
    }

    /**
     * Load conf into obj
     */
    private void loadConfig() throws DriverException {
        this.config = ConfigManager.getConfig();

        this.setMinWorkers(config.DOCKER_MIN_CONTAINERS);
        this.setMaxWorkers(config.DOCKER_MAX_CONTAINERS);
        this.setMaxBootsInLoop(config.DOCKER_MAX_BOOTS_IN_LOOP);
        this.setMaxShutdownsInLoop(config.DOCKER_MAX_SHUTDOWNS_IN_LOOP);
        this.setDockerEnvironment(config.DOCKER_ENV_VARS);
        this.setDockerServiceName(config.DOCKER_SERVICE_NAME);

        this.isLoaded = true;
    }

    /**
     * Get registered worker by worker name
     * @param name String
     * @return Worker
     */
    public Worker getWorker(String name) {
        return this.workers.get(name);
    }

    /**
     * Load workers from "docker-machine ls" into application
     */
    public void loadWorkers() {
        try {

            // @TODO: Load workers
            List<Container> currentContainers = dockerClient.listContainers(
                    DockerClient.ListContainersParam.allContainers(),
                    DockerClient.ListContainersParam.withLabel("environment", this.dockerEnvironment),
                    DockerClient.ListContainersParam.withLabel("service", this.dockerServiceName));

            Debugger.getInstance().print("Loaded " + currentContainers.size() + " containers matching ENV [" + this.dockerEnvironment + "] and Service [" + this.dockerServiceName,this.getClass());

            for (Container container: currentContainers) {
                // @TODO this.registerWorker(container.id(),container.id(),container.image(),container.ports(),)
            }





        } catch (Exception e) {
            System.out.println(Debugger.PREPEND_OUTPUT_DOCKER + "EXCEPTION: " + e.getMessage());
            e.printStackTrace();
        }


    }

    /**
     * Registers a new worker into the application
     * @return Boolean
     */
    public Boolean registerWorker(String id, String name, String image, ArrayList<String> bindPorts, ArrayList<String> envVars, ArrayList<String> mountedVolumes, ContainerState state) {
        try {
            Worker worker = new Worker(this, id, this.dockerClient, name, image, bindPorts, envVars, mountedVolumes, state);
            Boolean workerExists = this.workers.get(name) != null;
            if(workerExists) {
                this.workers.replace(name, worker);
            } else {
                this.workers.put(name, worker);
            }

            return true;
        } catch (Exception e) {
            System.out.println(Debugger.PREPEND_OUTPUT_DOCKER + e.getMessage());
            return false;
        }
    }

    /**
     * Provision a new worker into the Docker cluster
     * @param workerName String
     */
    public void provisionWorker(String workerName) {
        try {
            Debugger.getInstance().printOutput("Provisioning new worker #"+workerName);

            //Provision new worker (docker container run?)


            final HostConfig.Builder hostConfigBuilder = HostConfig.builder();

            // Set exposed ports @TODO
//            for (String volume:this.getConfig().DOCKER_PORTS) {
//                hostConfigBuilder.portb(volume);
//            }


            // Set volumes
            for (String volume:this.getConfig().DOCKER_VOLUMES) {
                hostConfigBuilder.appendBinds(volume);
            }


            ContainerConfig.Builder containerBuilder = ContainerConfig.builder()
                .image(this.getConfig().DOCKER_IMAGE)
                .env(config.DOCKER_ENV_VARS)
                .labels(ImmutableMap.of("environment",this.dockerEnvironment,"service",this.dockerServiceName))
                .hostConfig(hostConfigBuilder.build())
                .hostname(workerName);


            final ContainerCreation container = this.dockerClient.createContainer(containerBuilder.build(),workerName);

            ContainerInfo containerInfo = dockerClient.inspectContainer(container.id());
{'fraction_non_alphanumeric': 0.10194547102712502, 'fraction_numerical': 0.0006973014434139878, 'mean_word_length': 2.801219188974291, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3144868', 'n_tokens_mistral': 4011, 'n_tokens_neox': 3803, 'n_words': 745}