<commit_before>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
############


import time

from cloudify_rest_client.executions import Execution

from .exceptions import (ExecutionTimeoutError,
                         EventProcessingTimeoutError)


WAIT_FOR_EXECUTION_SLEEP_INTERVAL = 3
WORKFLOW_END_TYPES = {u'workflow_succeeded', u'workflow_failed',
                      u'workflow_cancelled'}


class ExecutionEventsFetcher(object):

    CONTEXT_FIELDS = [
        'deployment_id',
        'execution_id',
        'node_name',
        'operation',
        'workflow_id',
    ]

    def __init__(self,
                 client,
                 execution_id,
                 batch_size=100,
                 include_logs=False):
        self._client = client
        self._execution_id = execution_id
        self._batch_size = batch_size
        self._from_event = 0
        self._include_logs = include_logs
        # make sure execution exists before proceeding
        # a 404 will be raised otherwise
        self._client.executions.get(execution_id)

    def fetch_and_process_events_batch(self,
                                       events_handler=None,
                                       offset=None,
                                       size=None):
        events_list_response = self._fetch_events_batch(offset, size)
        total_events = events_list_response.metadata.pagination.total
        events = [
            self._map_api_event_to_internal_event(event)
            for event in events_list_response.items
        ]
        if events and events_handler:
            events_handler(events)

        return len(events), total_events

    def _fetch_events_batch(self, offset=None, size=None):
        offset = offset if offset is not None else self._from_event
        size = size if size is not None else self._batch_size
        events_list_response = self._client.events.list(
            execution_id=self._execution_id,
            _offset=offset,
            _size=size,
            include_logs=self._include_logs,
            sort='reported_timestamp')
        self._from_event += len(events_list_response)
        return events_list_response

    def _map_api_event_to_internal_event(self, event):
        """Map data structure from API to internal.

        This method adapts the data structure returend by the events API
        endpoint to the structure expected by `cloudify.event.Event`.

        Note: the event is modified in place, so even though the value is
        returned, the original data structure is not preserved.

        :param event: Event in API format
        :type event: dict(str)
        :return: Event in internal format
        :rtype: dict(str)

        """
        event['context'] = {
            context_field: event[context_field]
            for context_field in self.CONTEXT_FIELDS
        }
        for context_field in self.CONTEXT_FIELDS:
            del event[context_field]

        event['context']['node_id'] = event['node_instance_id']
        event['context']['source_id'] = event['source_id']
        event['context']['target_id'] = event['target_id']
        del event['node_instance_id']

        event['message'] = {
            'arguments': None,
            'text': event['message'],
        }

        event['context']['task_error_causes'] = event['error_causes']
        del event['error_causes']

        return event<commit_msg>Validate the key exists before assigning value (#906)<commit_after>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
############


import time

from cloudify_rest_client.executions import Execution

from .exceptions import (ExecutionTimeoutError,
                         EventProcessingTimeoutError)


WAIT_FOR_EXECUTION_SLEEP_INTERVAL = 3
WORKFLOW_END_TYPES = {u'workflow_succeeded', u'workflow_failed',
                      u'workflow_cancelled'}


class ExecutionEventsFetcher(object):

    CONTEXT_FIELDS = [
        'deployment_id',
        'execution_id',
        'node_name',
        'operation',
        'workflow_id',
    ]

    def __init__(self,
                 client,
                 execution_id,
                 batch_size=100,
                 include_logs=False):
        self._client = client
        self._execution_id = execution_id
        self._batch_size = batch_size
        self._from_event = 0
        self._include_logs = include_logs
        # make sure execution exists before proceeding
        # a 404 will be raised otherwise
        self._client.executions.get(execution_id)

    def fetch_and_process_events_batch(self,
                                       events_handler=None,
                                       offset=None,
                                       size=None):
        events_list_response = self._fetch_events_batch(offset, size)
        total_events = events_list_response.metadata.pagination.total
        events = [
            self._map_api_event_to_internal_event(event)
            for event in events_list_response.items
        ]
        if events and events_handler:
            events_handler(events)

        return len(events), total_events

    def _fetch_events_batch(self, offset=None, size=None):
        offset = offset if offset is not None else self._from_event
        size = size if size is not None else self._batch_size
        events_list_response = self._client.events.list(
            execution_id=self._execution_id,
            _offset=offset,
            _size=size,
            include_logs=self._include_logs,
            sort='reported_timestamp')
        self._from_event += len(events_list_response)
        return events_list_response

    def _map_api_event_to_internal_event(self, event):
        """Map data structure from API to internal.

        This method adapts the data structure returend by the events API
        endpoint to the structure expected by `cloudify.event.Event`.

        Note: the event is modified in place, so even though the value is
        returned, the original data structure is not preserved.

        :param event: Event in API format
        :type event: dict(str)
        :return: Event in internal format
        :rtype: dict(str)

        """
        event['context'] = {
            context_field: event[context_field]
            for context_field in self.CONTEXT_FIELDS
        }
        for context_field in self.CONTEXT_FIELDS:
            del event[context_field]

        event['context']['node_id'] = event['node_instance_id']
        if 'source_id' in event:
            event['context']['source_id'] = event['source_id']
            event['context']['target_id'] = event['target_id']
        del event['node_instance_id']

        event['message'] = {
            'arguments': None,
            'text': event['message'],
        }

        event['context']['task_error_causes'] = event['error_causes']
        del event['error_causes']

        return event
{'fraction_non_alphanumeric': 0.07967338859334405, 'fraction_numerical': 0.004330075467029568, 'mean_word_length': 2.346026490066225, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '118328', 'n_tokens_mistral': 2169, 'n_tokens_neox': 2084, 'n_words': 602}