<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
This script creates a HadoopYarn cluster on ~okeanos.

@author: <NAME>, <NAME>, <NAME>
"""
import datetime
from time import sleep
import logging
import subprocess
import json
from os.path import join, expanduser
from reroute_ssh import reroute_ssh_prep
from kamaki.clients import ClientError
from run_ansible_playbooks import install_yarn
from okeanos_utils import Cluster, check_credentials, endpoints_and_user_id, \
    init_cyclades, init_cyclades_netclient, init_plankton, get_project_id, \
    destroy_cluster, get_user_quota, set_cluster_state, retrieve_pending_clusters
from django_db_after_login import db_cluster_create
from cluster_errors_constants import *
from celery import current_task
import os


class YarnCluster(object):
    """
    Class for create hadoop-yarn cluster functionality
    """

    def __init__(self, opts):
        """Initialization of YarnCluster data attributes"""
        self.opts = opts
        # Master VM ip, placeholder value
        self.HOSTNAME_MASTER_IP = '127.0.0.1'
        # master VM root password file, placeholder value
        self.pass_file = 'PLACEHOLDER'
        self.hadoop_image = False
        # List of cluster VMs
        self.server_dict = {}
        if self.opts['disk_template'] == 'Archipelago':
            self.opts['disk_template'] = 'ext_vlmc'
        elif self.opts['disk_template'] == 'Standard':
            self.opts['disk_template'] = 'drbd'
        # project id of project name given as argument
        self.project_id = get_project_id(self.opts['token'],
                                         self.opts['project_name'])
        self.status = {}
        # Instance of an AstakosClient object
        self.auth = check_credentials(self.opts['token'],
                                      self.opts.get('auth_url',
                                                    auth_url))
        # Check if project has actual quota
        if self.check_project_quota() != 0:
            msg = 'Project %s exists but you have no quota to request' % \
                self.opts['project_name']
            raise ClientError(msg, error_project_quota)
        # ~okeanos endpoints and user id
        self.endpoints, self.user_id = endpoints_and_user_id(self.auth)

        # Instance of CycladesClient
        self.cyclades = init_cyclades(self.endpoints['cyclades'],
                                      self.opts['token'])
        # Instance of CycladesNetworkClient
        self.net_client = init_cyclades_netclient(self.endpoints['network'],
                                                  self.opts['token'])
        # Instance of Plankton/ImageClient
        self.plankton = init_plankton(self.endpoints['plankton'],
                                      self.opts['token'])
        # Get resources of pending clusters
        self.pending_quota = retrieve_pending_clusters(self.opts['token'],
                                                       self.opts['project_name'])
        # check escienceconf flag and set hadoop_image accordingly
        list_current_images = self.plankton.list_public(True, 'default')
        for image in list_current_images:
            if self.opts['os_choice'] == image['name']:
                try:
                    if image['properties']['escienceconf']:
                        image_metadata = json.loads(image['properties']['escienceconf'])
                        if image_metadata['ecosystem'] == 'True':
                            self.hadoop_image = 'ecosystem'
                        if 'cloudera' in image_metadata:
                            if image_metadata['cloudera'] == 'True':
                                self.hadoop_image = 'cloudera'
                        elif image_metadata['hadoop'] == 'True' and image_metadata['hue'] == 'True':
                            self.hadoop_image = 'hue'
                        elif image_metadata['hadoop'] == 'False':
                            self.hadoop_image = 'debianbase'
                        else:
                            self.hadoop_image = 'hadoopbase'

                except:
                    # if property hasn't been set then hadoop_image flag is false
                    self.hadoop_image = 'debianbase'

        self._DispatchCheckers = {}
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_cluster_size_quotas
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_network_quotas
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_ip_quotas
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_cpu_valid
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_ram_valid
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_disk_valid

    def check_cluster_size_quotas(self):
        """
        Checks if the user quota is enough to create the requested number
        of VMs.
        """
        dict_quotas = get_user_quota(self.auth)
        pending_vm = self.pending_quota['VMs']
        limit_vm = dict_quotas[self.project_id]['cyclades.vm']['limit']<commit_msg>Temporary fix for empty image strange bug<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
This script creates a HadoopYarn cluster on ~okeanos.

@author: <NAME>, <NAME>, <NAME>
"""
import datetime
from time import sleep
import logging
import subprocess
import json
from os.path import join, expanduser
from reroute_ssh import reroute_ssh_prep
from kamaki.clients import ClientError
from run_ansible_playbooks import install_yarn
from okeanos_utils import Cluster, check_credentials, endpoints_and_user_id, \
    init_cyclades, init_cyclades_netclient, init_plankton, get_project_id, \
    destroy_cluster, get_user_quota, set_cluster_state, retrieve_pending_clusters
from django_db_after_login import db_cluster_create
from cluster_errors_constants import *
from celery import current_task
import os


class YarnCluster(object):
    """
    Class for create hadoop-yarn cluster functionality
    """

    def __init__(self, opts):
        """Initialization of YarnCluster data attributes"""
        self.opts = opts
        # Master VM ip, placeholder value
        self.HOSTNAME_MASTER_IP = '127.0.0.1'
        # master VM root password file, placeholder value
        self.pass_file = 'PLACEHOLDER'
        self.hadoop_image = False
        # List of cluster VMs
        self.server_dict = {}
        if self.opts['disk_template'] == 'Archipelago':
            self.opts['disk_template'] = 'ext_vlmc'
        elif self.opts['disk_template'] == 'Standard':
            self.opts['disk_template'] = 'drbd'
        # project id of project name given as argument
        self.project_id = get_project_id(self.opts['token'],
                                         self.opts['project_name'])
        self.status = {}
        # Instance of an AstakosClient object
        self.auth = check_credentials(self.opts['token'],
                                      self.opts.get('auth_url',
                                                    auth_url))
        # Check if project has actual quota
        if self.check_project_quota() != 0:
            msg = 'Project %s exists but you have no quota to request' % \
                self.opts['project_name']
            raise ClientError(msg, error_project_quota)
        # ~okeanos endpoints and user id
        self.endpoints, self.user_id = endpoints_and_user_id(self.auth)

        # Instance of CycladesClient
        self.cyclades = init_cyclades(self.endpoints['cyclades'],
                                      self.opts['token'])
        # Instance of CycladesNetworkClient
        self.net_client = init_cyclades_netclient(self.endpoints['network'],
                                                  self.opts['token'])
        # Instance of Plankton/ImageClient
        self.plankton = init_plankton(self.endpoints['plankton'],
                                      self.opts['token'])
        # Get resources of pending clusters
        self.pending_quota = retrieve_pending_clusters(self.opts['token'],
                                                       self.opts['project_name'])
        # check escienceconf flag and set hadoop_image accordingly
        list_current_images = self.plankton.list_public(True, 'default')
        for image in list_current_images:
            if self.opts['os_choice'] == image['name']:
                if 'escienceconf' in image['properties']:
                    image_metadata = json.loads(image['properties']['escienceconf'])
                    if image_metadata['ecosystem'] == 'True':
                        self.hadoop_image = 'ecosystem'
                    elif image_metadata['cloudera'] == 'True':
                        self.hadoop_image = 'cloudera'
                    elif image_metadata['hadoop'] == 'True' and image_metadata['hue'] == 'True':
                        self.hadoop_image = 'hue'
                    else:
                        self.hadoop_image = 'hadoopbase'
                else:
                    self.hadoop_image = 'debianbase'
        if not self.hadoop_image:
            self.hadoop_image = 'debianbase'
        self._DispatchCheckers = {}
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_cluster_size_quotas
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_network_quotas
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_ip_quotas
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_cpu_valid
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_ram_valid
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_disk_valid

    def check_cluster_size_quotas(self):
        """
        Checks if the user quota is enough to create the requested number
        of VMs.
        """
        dict_quotas = get_user_quota(self.auth)
        pending_vm = self.pending_quota['VMs']
        limit_vm = dict_quotas[self.project_id]['cyclades.vm']['limit']
{'fraction_non_alphanumeric': 0.09181888845705402, 'fraction_numerical': 0.0027205596579867857, 'mean_word_length': 2.082659478885894, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6186544', 'n_tokens_mistral': 2803, 'n_tokens_neox': 2703, 'n_words': 608}