<commit_before># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: <NAME> (<EMAIL>)
#
from __future__ import unicode_literals
from __future__ import division

from fabric.api import settings as fabric_settings
from fabric.context_managers import cd, hide
from fabric.contrib import files as fabric_files
from fabric.operations import sudo, run, put, get
from fabric.state import env

from pyLibrary.debugs.logs import Log
from pyLibrary.env.files import File
from pyLibrary.maths import Math
from pyLibrary.maths.randoms import Random
from pyLibrary.meta import use_settings
from pyLibrary.strings import expand_template
from pyLibrary.thread.threads import Lock
from spot.instance_manager import InstanceManager


class ESSpot(InstanceManager):
    """
    THIS CLASS MUST HAVE AN IMPLEMENTATION FOR the SpotManager TO USE
    """
    @use_settings
    def __init__(self, settings):
        self.settings = settings
        self.conn = None
        self.instance = None
        self.locker = Lock()

    def required_utility(self):
        return self.settings.minimum_utility

    def setup(
        self,
        instance,   # THE boto INSTANCE OBJECT FOR THE MACHINE TO SETUP
        utility
    ):
        with self.locker:
            self.instance = instance
            gigabytes = Math.floor(utility, 15)
            Log.note("setup {{instance}}", instance=instance.id)
            with hide('output'):
                self._config_fabric(instance)
                self._install_es(gigabytes)
                self._install_indexer()
                self._install_supervisor()
                self._start_supervisor()

    def teardown(
        self,
        instance   # THE boto INSTANCE OBJECT FOR THE MACHINE TO TEARDOWN
    ):
        with self.locker:
            self.instance = instance
            Log.note("teardown {{instance}}", instance=instance.id)
            self._config_fabric(instance)
            # ASK NICELY TO STOP "supervisord" PROCESS
            with fabric_settings(warn_only=True):
                sudo("ps -ef | grep supervisord | grep -v grep | awk '{print $2}' | xargs kill -SIGINT")

            #WAIT FOR SUPERVISOR SHUTDOWN
            pid = True
            while pid:
                with hide('output'):
                    pid = sudo("ps -ef | grep supervisord | grep -v grep | awk '{print $2}'")

    def _config_fabric(self, instance):
        if not instance.ip_address:
            Log.error("Expecting an ip address for {{instance_id}}", instance_id=instance.id)

        for k, v in self.settings.connect.items():
            env[k] = v
        env.host_string = instance.ip_address
        env.abort_exception = Log.error

    def _install_es(self, gigabytes):
        volumes = self.instance.markup.drives

        if not fabric_files.exists("/usr/local/elasticsearch"):
            with cd("/home/ec2-user/"):
                run("mkdir -p temp")

            with cd("/home/ec2-user/temp"):
                run('wget -c --no-cookies --no-check-certificate --header "Cookie: s_cc=true; s_nr=1425654197863; s_sq=%5B%5BB%5D%5D; oraclelicense=accept-securebackup-cookie; gpw_e24=http%3A%2F%2Fwww.oracle.com%2Ftechnetwork%2Fjava%2Fjavase%2Fdownloads%2Fjre8-downloads-2133155.html" "http://download.oracle.com/otn-pub/java/jdk/8u40-b25/jre-8u40-linux-x64.rpm" --output-document="jdk-8u5-linux-x64.rpm"')
                sudo("rpm -i jdk-8u5-linux-x64.rpm")
                sudo("alternatives --install /usr/bin/java java /usr/java/default/bin/java 20000")
                run("export JAVA_HOME=/usr/java/default")

            with cd("/home/ec2-user/"):
                run('wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.4.2.tar.gz')
                run('tar zxfv elasticsearch-1.4.2.tar.gz')
                sudo('mkdir /usr/local/elasticsearch')
                sudo('cp -R elasticsearch-1.4.2/* /usr/local/elasticsearch/')

            with cd('/usr/local/elasticsearch/'):
                # BE SURE TO MATCH THE PLUGLIN WITH ES VERSION
                # https://github.com/elasticsearch/elasticsearch-cloud-aws
                sudo('bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.4.1')

        self.conn = self.instance.connection
<commit_msg>was not copying config files<commit_after># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: <NAME> (<EMAIL>)
#
from __future__ import unicode_literals
from __future__ import division

from fabric.api import settings as fabric_settings
from fabric.context_managers import cd, hide
from fabric.contrib import files as fabric_files
from fabric.operations import sudo, run, put, get
from fabric.state import env

from pyLibrary.debugs.logs import Log
from pyLibrary.env.files import File
from pyLibrary.maths import Math
from pyLibrary.maths.randoms import Random
from pyLibrary.meta import use_settings
from pyLibrary.strings import expand_template
from pyLibrary.thread.threads import Lock
from spot.instance_manager import InstanceManager


class ESSpot(InstanceManager):
    """
    THIS CLASS MUST HAVE AN IMPLEMENTATION FOR the SpotManager TO USE
    """
    @use_settings
    def __init__(self, settings):
        self.settings = settings
        self.conn = None
        self.instance = None
        self.locker = Lock()

    def required_utility(self):
        return self.settings.minimum_utility

    def setup(
        self,
        instance,   # THE boto INSTANCE OBJECT FOR THE MACHINE TO SETUP
        utility
    ):
        with self.locker:
            self.instance = instance
            gigabytes = Math.floor(utility, 15)
            Log.note("setup {{instance}}", instance=instance.id)
            with hide('output'):
                self._config_fabric(instance)
                self._install_es(gigabytes)
                self._install_indexer()
                self._install_supervisor()
                self._start_supervisor()

    def teardown(
        self,
        instance   # THE boto INSTANCE OBJECT FOR THE MACHINE TO TEARDOWN
    ):
        with self.locker:
            self.instance = instance
            Log.note("teardown {{instance}}", instance=instance.id)
            self._config_fabric(instance)
            # ASK NICELY TO STOP "supervisord" PROCESS
            with fabric_settings(warn_only=True):
                sudo("ps -ef | grep supervisord | grep -v grep | awk '{print $2}' | xargs kill -SIGINT")

            #WAIT FOR SUPERVISOR SHUTDOWN
            pid = True
            while pid:
                with hide('output'):
                    pid = sudo("ps -ef | grep supervisord | grep -v grep | awk '{print $2}'")

    def _config_fabric(self, instance):
        if not instance.ip_address:
            Log.error("Expecting an ip address for {{instance_id}}", instance_id=instance.id)

        for k, v in self.settings.connect.items():
            env[k] = v
        env.host_string = instance.ip_address
        env.abort_exception = Log.error

    def _install_es(self, gigabytes):
        volumes = self.instance.markup.drives

        if not fabric_files.exists("/usr/local/elasticsearch"):
            with cd("/home/ec2-user/"):
                run("mkdir -p temp")

            with cd("/home/ec2-user/temp"):
                run('wget -c --no-cookies --no-check-certificate --header "Cookie: s_cc=true; s_nr=1425654197863; s_sq=%5B%5BB%5D%5D; oraclelicense=accept-securebackup-cookie; gpw_e24=http%3A%2F%2Fwww.oracle.com%2Ftechnetwork%2Fjava%2Fjavase%2Fdownloads%2Fjre8-downloads-2133155.html" "http://download.oracle.com/otn-pub/java/jdk/8u40-b25/jre-8u40-linux-x64.rpm" --output-document="jdk-8u5-linux-x64.rpm"')
                sudo("rpm -i jdk-8u5-linux-x64.rpm")
                sudo("alternatives --install /usr/bin/java java /usr/java/default/bin/java 20000")
                run("export JAVA_HOME=/usr/java/default")

            with cd("/home/ec2-user/"):
                run('wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.4.2.tar.gz')
                run('tar zxfv elasticsearch-1.4.2.tar.gz')
                sudo('mkdir /usr/local/elasticsearch')
                sudo('cp -R elasticsearch-1.4.2/* /usr/local/elasticsearch/')

            with cd('/usr/local/elasticsearch/'):
                # BE SURE TO MATCH THE PLUGLIN WITH ES VERSION
                # https://github.com/elasticsearch/elasticsearch-cloud-aws
                sudo('bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.4.1')

            #REMOVE THESE FILES, WE WILL REPLACE THEM WITH THE CORRECT VERSIONS AT THE END
            sudo("rm -f /usr/local/elasticsearch/config/elasticsearch.yml")
            sudo("rm -f /usr/local/elasticsearch/bin/elasticsearch.in.sh")

        self.conn = self.instance.connection

{'fraction_non_alphanumeric': 0.10585783238262396, 'fraction_numerical': 0.017990346643264588, 'mean_word_length': 2.3704251386321626, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3252837', 'n_tokens_mistral': 2851, 'n_tokens_neox': 2690, 'n_words': 672}