<commit_before>#!/usr/bin/python
#
#    OpenStack Heat Plugin for interfacing with VMware Big Data Extensions
#
#    <NAME> - <EMAIL>
#    http://www.VirtualElephant.com
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import time, json, base64, requests, subprocess

from heat.engine import constraints, properties, resource
from heat.openstack.common import log as logging

logger = logging.getLogger(__name__)

class BigDataExtensions(resource.Resource):

    PROPERTIES = (
        BDE_ENDPOINT, VCM_SERVER, USERNAME, PASSWORD,
        CLUSTER_NAME, CLUSTER_TYPE, CLUSTER_NET, CLUSTER_PASSWORD
    ) = (
        'bde_endpoint', 'vcm_server', 'username', 'password',
        'cluster_name', 'cluster_type', 'cluster_net', 'cluster_password'
    )

    properties_schema = {
        BDE_ENDPOINT: properties.Schema(
            properties.Schema.STRING,
            required=True,
            default='bde.localdomain'
        ),
        VCM_SERVER: properties.Schema(
            properties.Schema.STRING,
            required=True,
            default='vcenter.localdomain'
        ),
        USERNAME: properties.Schema(
            properties.Schema.STRING,
            required=True,
            default='administrator@vsphere.local'
        ),
        PASSWORD: properties.Schema(
            properties.Schema.STRING,
            required=True,
            default='password'
        ),
        CLUSTER_NAME: properties.Schema(
            properties.Schema.STRING,
            required=True
        ),
        CLUSTER_TYPE: properties.Schema(
            properties.Schema.STRING,
            required=True
        ),
        CLUSTER_NET: properties.Schema(
            properties.Schema.STRING,
            required=True
        ),
        CLUSTER_PASSWORD: properties.Schema(
            properties.Schema.STRING,
            required=False
        )
    }

    def _open_connection(self):
        bde_server = self.properties.get(self.BDE_ENDPOINT)
        bde_user = self.properties.get(self.USERNAME)
        bde_pass = self.properties.get(self.PASSWORD)

        header = {'content-type': 'application/x-www-form-urlencoded'}
        prefix = 'https://'
        port = ':8443'
        auth_string = "/serengeti/j_spring_security_check"
        data = 'j_username=' + bde_user + '&j_password=' + bde_pass

        s = requests.session()
        url = prefix + bde_server + port + auth_string
        r = s.post(url, data, headers=header, verify=False)
        logger.info(_("VirtualElephant::VMware::BDE - Authentication status code %s") % r.json)

        return s

    def _close_connection(self):
        bde_server = self.properties.get(self.BDE_ENDPOINT)
        header = {'content-type': 'application/x-www-form-urlencoded'}
        url = 'https://' + bde_server + ':8443/serengeti/j_spring_security_logout'
        s = requests.session()
        r = s.post(url, headers=header, verify=False)
        logger.info(_("VirtualElephant::VMware::BDE - Log out status code %s") % r.json)
    
        return

    def handle_create(self):
        # REST API call to create a new VMware BDE cluster
        bde_server = self.properties.get(self.BDE_ENDPOINT)
        vcm_server = self.properties.get(self.VCM_SERVER)
        bde_user = self.properties.get(self.USERNAME)
        bde_pass = self.properties.get(self.PASSWORD)
        distro = self.properties.get(self.CLUSTER_TYPE)
        name = self.properties.get(self.CLUSTER_NAME)
        network = self.properties.get(self.CLUSTER_NET)
        prefix = 'https://'
        port = ':8443'

        # hack because of Heat sends call before NSX network is created/assigned
        #time.sleep(120)

        # determine actual NSX portgroup created
        # hack - regex in Python is not a strength
        mob_string = '/mob/?moid=datacenter-2'
        curl_cmd = 'curl -k -u ' + bde_user + ':' + bde_pass + ' ' + prefix + vcm_server + mob_string
        grep_cmd = " | grep -oP '(?<=\(vxw).*(?=" + network + "\))' | grep -oE '[^\(]+$'"
        awk_cmd = " | awk '{print $0 \"" + network + "\"}'"
        full_cmd = curl_cmd + grep_cmd + awk_cmd

        p = subprocess.Popen(full_cmd, stdout=subprocess.PIPE, shell=True)
        (net_uid, err) = p.communicate()

        # Check to see if network_id is as we expect it
        if 'vxw' in net_uid:
            network_id = net_uid
        else:
            network_id = "vxw" + net_uid

        network_id = network_id.rstrip('\n')

        # Authenticate in a requests.session to the BDE server
        curr = self._open_connection()

        # Should check to see if network already exists as available network
        # This logs a big fat error message in /opt/serengeti/logs/serengeti.log
        # when the network doesn't exist.
        header = {'content-type': 'application/json'}
        api_call = '/serengeti/api/network/' + network
        url = prefix + bde_server + port + api_call
        r = curr.get(url, headers=header, verify=False)

        # Add new network to BDE as an available network if check fails
        payload = {"name" : network, "portGroup" : network_id, "isDhcp" : "true"}
        api_call = '/serengeti/api/networks'
        url = prefix + bde_server + port + api_call
        r = curr.post(url, data=json.dumps(payload), headers=header, verify=False)
        logger.info(_("VirtualElephant::VMware::BDE - Network creation status code %s") % r.json)

        # Send the create cluster REST API call
        payload = {"name": name, "distro": distro, "networkConfig": { "MGT_NETWORK": [network]}}
        api_call = '/serengeti/api/clusters'
        url = prefix + bde_server + port + api_call
        r = curr.post(url, data=json.dumps(payload), headers=header, verify=False)
        logger.info(_("VirtualElephant::VMware::BDE - Create cluster status code %s") % r.json)

        term = self._close_connection()
        return

    def handle_suspend(self):
        # REST API call to shutdown an existing VMware BDE cluster
        bde_server = self.properties.get(self.BDE_ENDPOINT)
        bde_user = self.properties.get(self.USERNAME)
        bde_pass = self.properties.get(self.PASSWORD)<commit_msg>Minor change to allow resource pool variable to be used.<commit_after>#!/usr/bin/python
#
#    OpenStack Heat Plugin for interfacing with VMware Big Data Extensions
#
#    <NAME> - <EMAIL>
#    http://www.VirtualElephant.com
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import time, json, base64, requests, subprocess

from heat.engine import constraints, properties, resource
from heat.openstack.common import log as logging

logger = logging.getLogger(__name__)

class BigDataExtensions(resource.Resource):

    PROPERTIES = (
        BDE_ENDPOINT, VCM_SERVER, USERNAME, PASSWORD,
        CLUSTER_NAME, CLUSTER_TYPE, CLUSTER_NET, CLUSTER_PASSWORD, CLUSTER_RP
    ) = (
        'bde_endpoint', 'vcm_server', 'username', 'password',
        'cluster_name', 'cluster_type', 'cluster_net', 'cluster_password', 'cluster_rp'
    )

    properties_schema = {
        BDE_ENDPOINT: properties.Schema(
            properties.Schema.STRING,
            required=True,
            default='bde.localdomain'
        ),
        VCM_SERVER: properties.Schema(
            properties.Schema.STRING,
            required=True,
            default='vcenter.localdomain'
        ),
        USERNAME: properties.Schema(
            properties.Schema.STRING,
            required=True,
            default='<EMAIL>@<EMAIL>'
        ),
        PASSWORD: properties.Schema(
            properties.Schema.STRING,
            required=True,
            default='password'
        ),
        CLUSTER_NAME: properties.Schema(
            properties.Schema.STRING,
            required=True
        ),
        CLUSTER_TYPE: properties.Schema(
            properties.Schema.STRING,
            required=True
        ),
        CLUSTER_NET: properties.Schema(
            properties.Schema.STRING,
            required=True
        ),
        CLUSTER_PASSWORD: properties.Schema(
            properties.Schema.STRING,
            required=False
        ),
        CLUSTER_RP: properties.Schema(
            properties.Schema.STRING,
            required=True,
            default='openstackRP'
        )
    }

    def _open_connection(self):
        bde_server = self.properties.get(self.BDE_ENDPOINT)
        bde_user = self.properties.get(self.USERNAME)
        bde_pass = self.properties.get(self.PASSWORD)

        header = {'content-type': 'application/x-www-form-urlencoded'}
        prefix = 'https://'
        port = ':8443'
        auth_string = "/serengeti/j_spring_security_check"
        data = 'j_username=' + bde_user + '&j_password=' + bde_pass

        s = requests.session()
        url = prefix + bde_server + port + auth_string
        r = s.post(url, data, headers=header, verify=False)
        logger.info(_("VirtualElephant::VMware::BDE - Authentication status code %s") % r.json)

        return s

    def _close_connection(self):
        bde_server = self.properties.get(self.BDE_ENDPOINT)
        header = {'content-type': 'application/x-www-form-urlencoded'}
        url = 'https://' + bde_server + ':8443/serengeti/j_spring_security_logout'
        s = requests.session()
        r = s.post(url, headers=header, verify=False)
        logger.info(_("VirtualElephant::VMware::BDE - Log out status code %s") % r.json)
    
        return

    def handle_create(self):
        # REST API call to create a new VMware BDE cluster
        bde_server = self.properties.get(self.BDE_ENDPOINT)
        vcm_server = self.properties.get(self.VCM_SERVER)
        bde_user = self.properties.get(self.USERNAME)
        bde_pass = self.properties.get(self.PASSWORD)
        distro = self.properties.get(self.CLUSTER_TYPE)
        name = self.properties.get(self.CLUSTER_NAME)
        network = self.properties.get(self.CLUSTER_NET)
        rp = self.properties.get(self.CLUSTER_RP)
        prefix = 'https://'
        port = ':8443'

        # hack because of Heat sends call before NSX network is created/assigned
        time.sleep(30)

        # determine actual NSX portgroup created
        # hack - regex in Python is not a strength
        mob_string = '/mob/?moid=datacenter-2'
        curl_cmd = 'curl -k -u ' + bde_user + ':' + bde_pass + ' ' + prefix + vcm_server + mob_string
        grep_cmd = " | grep -oP '(?<=\(vxw).*(?=" + network + "\))' | grep -oE '[^\(]+$'"
        awk_cmd = " | awk '{print $0 \"" + network + "\"}'"
        full_cmd = curl_cmd + grep_cmd + awk_cmd

        p = subprocess.Popen(full_cmd, stdout=subprocess.PIPE, shell=True)
        (net_uid, err) = p.communicate()

        # Check to see if network_id is as we expect it
        if 'vxw' in net_uid:
            network_id = net_uid
        else:
            network_id = "vxw" + net_uid

        network_id = network_id.rstrip('\n')

        # Authenticate in a requests.session to the BDE server
        curr = self._open_connection()

        # Should check to see if network already exists as available network
        # This logs a big fat error message in /opt/serengeti/logs/serengeti.log
        # when the network doesn't exist.
        header = {'content-type': 'application/json'}
        api_call = '/serengeti/api/network/' + network
        url = prefix + bde_server + port + api_call
        r = curr.get(url, headers=header, verify=False)

        # Add new network to BDE as an available network if check fails
        payload = {"name" : network, "portGroup" : network_id, "isDhcp" : "true"}
        api_call = '/serengeti/api/networks'
        url = prefix + bde_server + port + api_call
        r = curr.post(url, data=json.dumps(payload), headers=header, verify=False)
        logger.info(_("VirtualElephant::VMware::BDE - Network creation status code %s") % r.json)

        # Send the create cluster REST API call
        payload = {"name": name, "distro": distro, "rpNames": [rp],  "networkConfig": { "MGT_NETWORK": [network]}}
        api_call = '/serengeti/api/clusters'
        url = prefix + bde_server + port + api_call
        r = curr.post(url, data=json.dumps(payload), headers=header, verify=False)
        logger.info(_("VirtualElephant::VMware::BDE - Create cluster status code %s") % r.json)

        term = self._close_connection()
        return

    def handle_suspend(self):
        # REST API call to shutdown an existing VMware BDE cluster
        bde_server = self.properties.get(self.BDE_ENDPOINT)
        bde_user = self.properties.get(self.USERNAME)
        bde_pass = self.properties.get(self.PASSWORD)
{'fraction_non_alphanumeric': 0.10896025810236105, 'fraction_numerical': 0.0032996040475142983, 'mean_word_length': 2.456411556006082, 'pattern_counts': {'":': 9, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 6, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4010341', 'n_tokens_mistral': 4146, 'n_tokens_neox': 4047, 'n_words': 997}