<commit_before># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: <NAME> (<EMAIL>)
#
from __future__ import unicode_literals
from __future__ import division

from fabric.api import settings as fabric_settings
from fabric.context_managers import cd, hide
from fabric.contrib import files as fabric_files
from fabric.operations import run, sudo, put
from fabric.state import env

from pyLibrary.debugs import constants
from pyLibrary.debugs import startup

from pyLibrary import aws
from pyLibrary.debugs.logs import Log
from pyLibrary.env.files import File
from pyLibrary.maths import Math
from pyLibrary.meta import use_settings
from pyLibrary.strings import between
from pyLibrary.thread.threads import Lock
from spot.instance_manager import InstanceManager


class ETL(InstanceManager):
    @use_settings
    def __init__(
        self,
        work_queue,  # SETTINGS FOR AWS QUEUE
        connect,  # SETTINGS FOR Fabric `env` TO CONNECT TO INSTANCE
        minimum_utility,
        settings=None
    ):
        InstanceManager.__init__(self, settings)
        self.locker = Lock()
        self.settings = settings

    def required_utility(self):
        queue = aws.Queue(self.settings.work_queue)
        pending = len(queue)
        return max(self.settings.minimum_utility, Math.ceiling(pending / 30))

    def setup(self, instance, utility):
        with self.locker:
            cpu_count = int(round(utility))

            with hide('output'):
                Log.note("setup {{instance}}", instance=instance.id)
                self._config_fabric(instance)
                Log.note("setup etl on {{instance}}", instance=instance.id)
                self._setup_etl_code()
                Log.note("add config file on {{instance}}", instance=instance.id)
                self._add_private_file()
                Log.note("setup supervisor on {{instance}}", instance=instance.id)
                self._setup_etl_supervisor(cpu_count)
                Log.note("setup done {{instance}}", instance=instance.id)

    def teardown(self, instance):
        with self.locker:
            Log.note("teardown {{instance}}", instance=instance.id)
            self._config_fabric(instance)
            sudo("supervisorctl stop all")

    def _setup_etl_code(self):
        sudo("dpkg --configure -a")
        sudo("apt-get update")
        sudo("apt-get clean")
        sudo("apt-get install -y python2.7")

        if not fabric_files.exists("/usr/local/bin/pip"):
            run("mkdir -p /home/ubuntu/temp")

            with cd("/home/ubuntu/temp"):
                # INSTALL FROM CLEAN DIRECTORY
                run("wget https://bootstrap.pypa.io/get-pip.py")
                sudo("rm -fr ~/.cache/pip")  # JUST IN CASE THE DIRECTORY WAS MADE
                sudo("python get-pip.py")

        if not fabric_files.exists("/home/ubuntu/ActiveData-ETL/README.md"):
            with cd("/home/ubuntu"):
                sudo("apt-get -yf install git-core")
                run('rm -fr /home/ubuntu/ActiveData-ETL')
                run("git clone https://github.com/klahnakoski/ActiveData-ETL.git")
                run("mkdir -p /home/ubuntu/ActiveData-ETL/results/logs")


        with cd("/home/ubuntu/ActiveData-ETL"):
            run("git checkout etl")
            # pip install -r requirements.txt HAS TROUBLE IMPORTING SOME LIBS
            sudo("pip install BeautifulSoup")
            sudo("pip install MozillaPulse")
            sudo("pip install boto")
            sudo("pip install requests")
            sudo("pip install taskcluster")
            sudo("apt-get -y install python-psycopg2")

    def _setup_etl_supervisor(self, cpu_count):
        # INSTALL supervsor
        sudo("apt-get install -y supervisor")
        with fabric_settings(warn_only=True):
            run("service supervisor start")<commit_msg>etl: Install lcov and use the codecoverage branch<commit_after># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: <NAME> (<EMAIL>)
#
from __future__ import unicode_literals
from __future__ import division

from fabric.api import settings as fabric_settings
from fabric.context_managers import cd, hide
from fabric.contrib import files as fabric_files
from fabric.operations import run, sudo, put
from fabric.state import env

from pyLibrary.debugs import constants
from pyLibrary.debugs import startup

from pyLibrary import aws
from pyLibrary.debugs.logs import Log
from pyLibrary.env.files import File
from pyLibrary.maths import Math
from pyLibrary.meta import use_settings
from pyLibrary.strings import between
from pyLibrary.thread.threads import Lock
from spot.instance_manager import InstanceManager


class ETL(InstanceManager):
    @use_settings
    def __init__(
        self,
        work_queue,  # SETTINGS FOR AWS QUEUE
        connect,  # SETTINGS FOR Fabric `env` TO CONNECT TO INSTANCE
        minimum_utility,
        settings=None
    ):
        InstanceManager.__init__(self, settings)
        self.locker = Lock()
        self.settings = settings

    def required_utility(self):
        queue = aws.Queue(self.settings.work_queue)
        pending = len(queue)
        return max(self.settings.minimum_utility, Math.ceiling(pending / 30))

    def setup(self, instance, utility):
        with self.locker:
            cpu_count = int(round(utility))

            with hide('output'):
                Log.note("setup {{instance}}", instance=instance.id)
                self._config_fabric(instance)
                Log.note("setup etl on {{instance}}", instance=instance.id)
                self._setup_etl_code()
                Log.note("add config file on {{instance}}", instance=instance.id)
                self._add_private_file()
                Log.note("setup supervisor on {{instance}}", instance=instance.id)
                self._setup_etl_supervisor(cpu_count)
                Log.note("setup done {{instance}}", instance=instance.id)

    def teardown(self, instance):
        with self.locker:
            Log.note("teardown {{instance}}", instance=instance.id)
            self._config_fabric(instance)
            sudo("supervisorctl stop all")

    def _setup_etl_code(self):
        sudo("dpkg --configure -a")
        sudo("apt-get update")
        sudo("apt-get clean")
        sudo("apt-get install -y python2.7 lcov")

        if not fabric_files.exists("/usr/local/bin/pip"):
            run("mkdir -p /home/ubuntu/temp")

            with cd("/home/ubuntu/temp"):
                # INSTALL FROM CLEAN DIRECTORY
                run("wget https://bootstrap.pypa.io/get-pip.py")
                sudo("rm -fr ~/.cache/pip")  # JUST IN CASE THE DIRECTORY WAS MADE
                sudo("python get-pip.py")

        if not fabric_files.exists("/home/ubuntu/ActiveData-ETL/README.md"):
            with cd("/home/ubuntu"):
                sudo("apt-get -yf install git-core")
                run('rm -fr /home/ubuntu/ActiveData-ETL')
                run("git clone https://github.com/klahnakoski/ActiveData-ETL.git")
                run("mkdir -p /home/ubuntu/ActiveData-ETL/results/logs")


        with cd("/home/ubuntu/ActiveData-ETL"):
            run("git checkout codecoverage")
            # pip install -r requirements.txt HAS TROUBLE IMPORTING SOME LIBS
            sudo("pip install BeautifulSoup")
            sudo("pip install MozillaPulse")
            sudo("pip install boto")
            sudo("pip install requests")
            sudo("pip install taskcluster")
            sudo("apt-get -y install python-psycopg2")

    def _setup_etl_supervisor(self, cpu_count):
        # INSTALL supervsor
        sudo("apt-get install -y supervisor")
        with fabric_settings(warn_only=True):
            run("service supervisor start")
{'fraction_non_alphanumeric': 0.10344400396432112, 'fraction_numerical': 0.002477700693756194, 'mean_word_length': 2.4397102684277803, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2796761', 'n_tokens_mistral': 2412, 'n_tokens_neox': 2231, 'n_words': 619}