<commit_before>########
# Copyright (c) 2013 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

__author__ = 'ran'

import tempfile
import os
import sys
import time
import shutil
import tarfile
import json

from os.path import expanduser
from contextlib import contextmanager
from urllib2 import HTTPError, URLError
from httplib import HTTPException

from swagger.swagger import ApiClient
from swagger.BlueprintsApi import BlueprintsApi
from swagger.ExecutionsApi import ExecutionsApi
from swagger.DeploymentsApi import DeploymentsApi
from swagger.events_api import EventsApi
from swagger.nodes_api import NodesApi


class CosmoManagerRestClient(object):

    def __init__(self, server_ip, port=8100):
        server_url = 'http://{0}:{1}'.format(server_ip, port)
        api_client = ApiClient(apiServer=server_url, apiKey='')
        self._blueprints_api = BlueprintsApi(api_client)
        self._executions_api = ExecutionsApi(api_client)
        self._deployments_api = DeploymentsApi(api_client)
        self._nodes_api = NodesApi(api_client)
        self._events_api = EventsApi(api_client)

    def list_blueprints(self):
        with self._protected_call_to_server('listing blueprints'):
            return self._blueprints_api.list()

    def publish_blueprint(self, blueprint_path):
        tempdir = tempfile.mkdtemp()
        try:
            tar_path = self._tar_blueprint(blueprint_path, tempdir)
            application_file = os.path.basename(blueprint_path)

            with self._protected_call_to_server('publishing blueprint'):
                with open(tar_path, 'rb') as f:
                    blueprint_state = self._blueprints_api.upload(
                        f,
                        application_file_name=application_file)
                return blueprint_state
        finally:
            shutil.rmtree(tempdir)

    def delete_blueprint(self, blueprint_id):
        with self._protected_call_to_server('deleting blueprint'):
            return self._blueprints_api.delete(blueprint_id)

    def validate_blueprint(self, blueprint_id):
        with self._protected_call_to_server('validating blueprint'):
            return self._blueprints_api.validate(blueprint_id)

    def list_deployments(self):
        with self._protected_call_to_server('list deployments'):
            return self._deployments_api.list()

    def create_deployment(self, blueprint_id):
        with self._protected_call_to_server('creating new deployment'):
            body = {
                'blueprintId': blueprint_id
            }
            return self._deployments_api.createDeployment(body=body)

    @staticmethod
    def _create_events_query(execution_id, timestamp=None):
        execution_id_match = {"match": {"context.execution_id": execution_id}}
        if timestamp is not None:
            query = {
                "bool": {
                    "must": [
                        execution_id_match,
                        {"range": {"@timestamp": {"gt": timestamp}}}
                    ]
                }
            }
        else:
            query = execution_id_match
        return query

    def get_execution_events(self,
                             execution_id,
                             timestamp=None,
                             from_event=0,
                             batch_size=100):
        """
        Returns event for the provided execution id.

        Args:
            execution_id: Id of execution to get events for.
            timestamp: The timestamp to get events after
                (None for getting all available events).
            from_event: Index of first event to retrieve on pagination.
            batch_size: Maximum number of events to retrieve per call.

        Returns (3 values):
            Events list, total number of currently available events and
            last event timestamp.
        """
        body = {
            "from": from_event,
            "size": batch_size,
            "sort": [{"@timestamp": {"order": "asc"}}],
            "query": self._create_events_query(execution_id, timestamp)
        }
        result = self.get_events(body)
        events = map(lambda x: x['_source'], result.json()['hits']['hits'])
        total_events = result.json()['hits']['total']
        last_timestamp = events[-1]['@timestamp'] if len(events) > 0 else None
        return events, total_events, last_timestamp

    def execute_deployment(self, deployment_id, operation, events_handler=None,
                           timeout=900):
        end = time.time() + timeout

        with self._protected_call_to_server('executing deployment operation'):
            body = {
                'workflowId': operation
            }
            execution = self._deployments_api.execute(
                deployment_id=deployment_id,
                body=body)

            end_states = ('terminated', 'failed')
            timestamp = None
            from_event = 0

            def handle_events(execution_events):
                if len(execution_events) > 0:
                    events_handler(execution_events)
                return len(execution_events)

            while execution.status not in end_states:
                if end < time.time():
                    raise CosmoManagerRestCallError(
                        'execution of operation {0} for deployment {1} timed '
                        'out'.format(operation, deployment_id))
                time.sleep(3)

                execution = self._executions_api.getById(execution.id)

                if events_handler is not None:
                    events, total_events, timestamp = \
                        self.get_execution_events(execution.id,
                                                  timestamp=timestamp,
                                                  from_event=from_event)
                    handle_events(events)

            # get remaining events after execution is over
            while events_handler is not None:
                time.sleep(1)
                events, total_events, timestamp = \
                    self.get_execution_events(execution.id,
                                              timestamp=timestamp,
                                              from_event=from_event)
                handle_events(events)
                if from_event >= total_events:
                    break

            if execution.status != 'terminated':
                raise CosmoManagerRestCallError('Execution of operation {0} '
                                                'for deployment {1} failed. '
                                                '[execution_id={2}, '
                                                'status_response={3}]'
                                                .format(
                                                    operation,
                                                    deployment_id,
                                                    execution.id,
                                                    execution.error))
<commit_msg>CLOUDIFY-2386 added timeout to events while loop<commit_after>########
# Copyright (c) 2013 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

__author__ = 'ran'

import tempfile
import os
import sys
import time
import shutil
import tarfile
import json

from os.path import expanduser
from contextlib import contextmanager
from urllib2 import HTTPError, URLError
from httplib import HTTPException

from swagger.swagger import ApiClient
from swagger.BlueprintsApi import BlueprintsApi
from swagger.ExecutionsApi import ExecutionsApi
from swagger.DeploymentsApi import DeploymentsApi
from swagger.events_api import EventsApi
from swagger.nodes_api import NodesApi


class CosmoManagerRestClient(object):

    def __init__(self, server_ip, port=8100):
        server_url = 'http://{0}:{1}'.format(server_ip, port)
        api_client = ApiClient(apiServer=server_url, apiKey='')
        self._blueprints_api = BlueprintsApi(api_client)
        self._executions_api = ExecutionsApi(api_client)
        self._deployments_api = DeploymentsApi(api_client)
        self._nodes_api = NodesApi(api_client)
        self._events_api = EventsApi(api_client)

    def list_blueprints(self):
        with self._protected_call_to_server('listing blueprints'):
            return self._blueprints_api.list()

    def publish_blueprint(self, blueprint_path):
        tempdir = tempfile.mkdtemp()
        try:
            tar_path = self._tar_blueprint(blueprint_path, tempdir)
            application_file = os.path.basename(blueprint_path)

            with self._protected_call_to_server('publishing blueprint'):
                with open(tar_path, 'rb') as f:
                    blueprint_state = self._blueprints_api.upload(
                        f,
                        application_file_name=application_file)
                return blueprint_state
        finally:
            shutil.rmtree(tempdir)

    def delete_blueprint(self, blueprint_id):
        with self._protected_call_to_server('deleting blueprint'):
            return self._blueprints_api.delete(blueprint_id)

    def validate_blueprint(self, blueprint_id):
        with self._protected_call_to_server('validating blueprint'):
            return self._blueprints_api.validate(blueprint_id)

    def list_deployments(self):
        with self._protected_call_to_server('list deployments'):
            return self._deployments_api.list()

    def create_deployment(self, blueprint_id):
        with self._protected_call_to_server('creating new deployment'):
            body = {
                'blueprintId': blueprint_id
            }
            return self._deployments_api.createDeployment(body=body)

    @staticmethod
    def _create_events_query(execution_id, timestamp=None):
        execution_id_match = {"match": {"context.execution_id": execution_id}}
        if timestamp is not None:
            query = {
                "bool": {
                    "must": [
                        execution_id_match,
                        {"range": {"@timestamp": {"gt": timestamp}}}
                    ]
                }
            }
        else:
            query = execution_id_match
        return query

    def get_execution_events(self,
                             execution_id,
                             timestamp=None,
                             from_event=0,
                             batch_size=100):
        """
        Returns event for the provided execution id.

        Args:
            execution_id: Id of execution to get events for.
            timestamp: The timestamp to get events after
                (None for getting all available events).
            from_event: Index of first event to retrieve on pagination.
            batch_size: Maximum number of events to retrieve per call.

        Returns (3 values):
            Events list, total number of currently available events and
            last event timestamp.
        """
        body = {
            "from": from_event,
            "size": batch_size,
            "sort": [{"@timestamp": {"order": "asc"}}],
            "query": self._create_events_query(execution_id, timestamp)
        }
        result = self.get_events(body)
        events = map(lambda x: x['_source'], result.json()['hits']['hits'])
        total_events = result.json()['hits']['total']
        last_timestamp = events[-1]['@timestamp'] if len(events) > 0 else None
        return events, total_events, last_timestamp

    def execute_deployment(self, deployment_id, operation, events_handler=None,
                           timeout=900):
        end = time.time() + timeout

        with self._protected_call_to_server('executing deployment operation'):
            body = {
                'workflowId': operation
            }
            execution = self._deployments_api.execute(
                deployment_id=deployment_id,
                body=body)

            end_states = ('terminated', 'failed')
            timestamp = None
            from_event = 0

            def handle_events(execution_events):
                if len(execution_events) > 0:
                    events_handler(execution_events)
                return len(execution_events)

            while execution.status not in end_states:
                if end < time.time():
                    raise CosmoManagerRestCallError(
                        'execution of operation {0} for deployment {1} timed '
                        'out'.format(operation, deployment_id))
                time.sleep(3)

                execution = self._executions_api.getById(execution.id)

                if events_handler is not None:
                    events, total_events, timestamp = \
                        self.get_execution_events(execution.id,
                                                  timestamp=timestamp,
                                                  from_event=from_event)
                    handle_events(events)

            # get remaining events after execution is over
            while events_handler is not None:
                if end < time.time():
                    raise CosmoManagerRestCallError(
                        'execution of operation {0} for deployment {1} timed '
                        'out'.format(operation, deployment_id))
                time.sleep(1)
                events, total_events, timestamp = \
                    self.get_execution_events(execution.id,
                                              timestamp=timestamp,
                                              from_event=from_event)
                handle_events(events)
                if from_event >= total_events:
                    break

            if execution.status != 'terminated':
                raise CosmoManagerRestCallError('Execution of operation {0} '
                                                'for deployment {1} failed. '
                                                '[execution_id={2}, '
                                                'status_response={3}]'
                                                .format(
                                                    operation,
                                                    deployment_id,
                                                    execution.id,
                                                    execution.error))

{'fraction_non_alphanumeric': 0.07405499388241354, 'fraction_numerical': 0.0048940691609247215, 'mean_word_length': 1.7117164309411559, 'pattern_counts': {'":': 26, '<': 6, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2976742', 'n_tokens_mistral': 3804, 'n_tokens_neox': 3610, 'n_words': 971}