<commit_before>#!/usr/bin/python
#
# Copyright 2015 The Cluster-Insight Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


"""Computes a context graph from raw context metadata.

The raw context metadata is gathered from a Kubernetes master and the Docker
daemons of its nodes.

The context graph is computed by a pool of concurrent worker threads.
The purpose of the worker threads is to reduce the elapsed time by
performing independent operations concurrently.

We store the outstanding operations in a random order in the input
work queue in order to prevent long trains of accessess to the same
system component (such as the Kubernetes master or a Docker controller
on a minion node).

There is no need to verify the existence of attributes in all wrapped
objects (the output of utilities.wrap_object()), because we assume that
the object was already verified by the corresponding get_xxx() routine
in kubernetes.py or docker.py.

The get_xxx() routine should have skipped any invalid object and not
return it to the caller.

Usage:

import context
import global_state
gs = global_state.GlobalState()
# initialize the global state in 'gs'.
# select the desired output format.
context.compute_graph(gs, output_format)
"""

import copy
import datetime
import Queue  # "Queue" was renamed "queue" in Python 3.
import re
import sys
import threading
import types

# local imports
import collector_error
import constants
import docker
import global_state
import kubernetes
import utilities


class ContextGraph(object):
  """Maintains the context graph and outputs it.

  This class is thread-safe.
  """

  def __init__(self):
    self._lock = threading.Lock()
    self._graph_metadata = None
    self._graph_title = None
    # Set the color lookup table of various resources.
    self._graph_color = {
        'Cluster': 'black',
        'Node': 'red',
        'Service': 'darkgreen',
        'ReplicationController': 'purple',
        'Pod': 'blue',
        'Container': 'green',
        'Process': 'gold',
        'Image': 'maroon'
    }
    self._context_resources = []
    self._context_relations = []
    self._version = None
    self._id_set = set()
    self._previous_relations_to_timestamps = {}
    self._current_relations_to_timestamps = {}

  def get_relations_to_timestamps(self):
    return self._current_relations_to_timestamps

  def set_relations_to_timestamps(self, d):
    assert isinstance(d, types.DictType)
    self._previous_relations_to_timestamps = d

  def add_resource(self, rid, annotations, rtype, timestamp, obj):
    """Adds a resource to the context graph."""
    assert utilities.valid_string(rid)
    assert utilities.valid_string(utilities.get_attribute(<commit_msg>Lock access to global state of ContextGraph.<commit_after>#!/usr/bin/python
#
# Copyright 2015 The Cluster-Insight Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


"""Computes a context graph from raw context metadata.

The raw context metadata is gathered from a Kubernetes master and the Docker
daemons of its nodes.

The context graph is computed by a pool of concurrent worker threads.
The purpose of the worker threads is to reduce the elapsed time by
performing independent operations concurrently.

We store the outstanding operations in a random order in the input
work queue in order to prevent long trains of accessess to the same
system component (such as the Kubernetes master or a Docker controller
on a minion node).

There is no need to verify the existence of attributes in all wrapped
objects (the output of utilities.wrap_object()), because we assume that
the object was already verified by the corresponding get_xxx() routine
in kubernetes.py or docker.py.

The get_xxx() routine should have skipped any invalid object and not
return it to the caller.

Usage:

import context
import global_state
gs = global_state.GlobalState()
# initialize the global state in 'gs'.
# select the desired output format.
context.compute_graph(gs, output_format)
"""

import copy
import datetime
import Queue  # "Queue" was renamed "queue" in Python 3.
import re
import sys
import threading
import types

# local imports
import collector_error
import constants
import docker
import global_state
import kubernetes
import utilities


class ContextGraph(object):
  """Maintains the context graph and outputs it.

  This class is thread-safe.
  """

  def __init__(self):
    self._lock = threading.Lock()
    self._graph_metadata = None
    self._graph_title = None
    # Set the color lookup table of various resources.
    self._graph_color = {
        'Cluster': 'black',
        'Node': 'red',
        'Service': 'darkgreen',
        'ReplicationController': 'purple',
        'Pod': 'blue',
        'Container': 'green',
        'Process': 'gold',
        'Image': 'maroon'
    }
    self._context_resources = []
    self._context_relations = []
    self._version = None
    self._id_set = set()
    self._previous_relations_to_timestamps = {}
    self._current_relations_to_timestamps = {}

  def get_relations_to_timestamps(self):
    with self._lock:
      return self._current_relations_to_timestamps

  def set_relations_to_timestamps(self, d):
    assert isinstance(d, types.DictType)
    with self._lock:
      self._previous_relations_to_timestamps = d

  def add_resource(self, rid, annotations, rtype, timestamp, obj):
    """Adds a resource to the context graph."""
    assert utilities.valid_string(rid)
    assert utilities.valid_string(utilities.get_attribute(
{'fraction_non_alphanumeric': 0.07181136120042872, 'fraction_numerical': 0.002756086357372531, 'mean_word_length': 4.024615384615385, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1263469', 'n_tokens_mistral': 1788, 'n_tokens_neox': 1708, 'n_words': 753}