<commit_before># Copyright 2013-2020 Aerospike, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy
import re
import threading
from time import time

from lib.client import util
from lib.client.node import Node
from lib.utils import util as commonutil
from lib.utils.constants import AuthMode
from lib.utils.lookupdict import LookupDict


# interval time in second for cluster refreshing
CLUSTER_REFRESH_INTERVAL = 3


class Cluster(object):
    # Kinda like a singleton... All instantiated classes will share the same
    # state.
    cluster_state = {}
    use_services_alumni = False
    use_services_alt = False
    crawl_lock = threading.Lock()

    def __init__(self, seed_nodes, user=None, password=<PASSWORD>, auth_mode=AuthMode.INTERNAL,
                 use_services_alumni=False, use_services_alt=False,
                 ssl_context=None, only_connect_seed=False, timeout=5):
        """
        Want to be able to support multiple nodes on one box (for testing)
        seed_nodes should be the form (address,port,tls) address can be fqdn or ip.
        """

        self.__dict__ = self.cluster_state
        if self.cluster_state != {}:
            return

        self._timeout = timeout

        self.user = user
        self.password = password
        self.auth_mode = auth_mode

        Cluster.use_services_alumni = use_services_alumni
        Cluster.use_services_alt = use_services_alt

        # self.nodes is a dict from Node ID -> Node objects
        self.nodes = {}

        # to avoid multiple entries of endpoints for same server we are keeping
        # this pointers
        self.aliases = {}

        # self.node_lookup is a dict of (fqdn, port) -> Node
        # and (ip, port) -> Node, and node.node_id -> Node
        self.node_lookup = LookupDict()

        self._seed_nodes = set(seed_nodes)
        self._live_nodes = set()
        self.ssl_context = ssl_context

        # crawl the cluster search for nodes in addition to the seed nodes.
        self.last_cluster_refresh_time = 0
        self.only_connect_seed = only_connect_seed
        self._refresh_cluster()

        # to avoid same label (NODE column) for multiple nodes we need to keep track
        # of available nodes name, if names are same then we can use ip:port
        self._same_name_nodes = False

    def __str__(self):
        nodes = self.nodes.values()
        if len(nodes) == 0:
            return ""

        online = [n.key for n in [n for n in nodes if n.alive]]
        offline = [n.key for n in [n for n in nodes if not n.alive]]

        retval = "Found %s nodes" % (len(nodes))
        if online:
            retval += "\nOnline:  %s" % (", ".join(online))
        if offline:
            retval += "\nOffline: %s" % (", ".join(offline))

        return retval

    def get_node_displaynames(self):
        node_names = {}
        for node_key, node in self.nodes.items():
            k = node.sock_name(use_fqdn=True)
            if commonutil.is_valid_ip_port(k):
                node_names[node_key] = k
            else:
                node_names[node_key] = self.node_lookup.get_shortname(k, min_prefix_len=20, min_suffix_len=5)

        return node_names

    def get_node_names(self):
        node_names = {}

        if not self._same_name_nodes:
            for node_key, node in self.nodes.items():
                name = node.sock_name(use_fqdn=True)
                if name in list(node_names.values()):
                    # found same name for multiple nodes
                    self._same_name_nodes = True
                    node_names.clear()
                    break
                node_names[node_key] = name

        if not node_names:
            for node_key, node in self.nodes.items():
                node_names[node_key] = node.sock_name(use_fqdn=False)

        return node_names

    def get_expected_principal(self):
        try:
            principal = "0"
            for k in self.nodes.keys():
                n = self.nodes[k]
                if n.node_id.zfill(16) > principal.zfill(16):
                    principal = n.node_id
            return principal
        except Exception as e:
            print(e)
            return ''

    def get_live_nodes(self):
        return self._live_nodes

    def get_visibility_error_nodes(self):
        visible = self.get_live_nodes()
        cluster_visibility_error_nodes = []
        for k in self.nodes.keys():
            node = self.nodes[k]
            if not node.alive:
                # in case of using alumni services, we might have offline nodes
                # which can't detect online nodes
                continue
            peers = util.flatten(node.peers)
            not_visible = set(visible) - set(peers)<commit_msg>Add cluster changes<commit_after># Copyright 2013-2020 Aerospike, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy
import re
import threading
import logging
from time import time

from lib.client import util
from lib.client.node import Node
from lib.utils import util as commonutil
from lib.utils.constants import AuthMode
from lib.utils.lookupdict import LookupDict


# interval time in second for cluster refreshing
CLUSTER_REFRESH_INTERVAL = 3


class Cluster(object):
    # Kinda like a singleton... All instantiated classes will share the same
    # state.
    cluster_state = {}
    use_services_alumni = False
    use_services_alt = False
    crawl_lock = threading.Lock()
    logger = logging.getLogger("asadm")

    def __init__(self, seed_nodes, user=None, password=<PASSWORD>, auth_mode=AuthMode.INTERNAL,
                 use_services_alumni=False, use_services_alt=False,
                 ssl_context=None, only_connect_seed=False, timeout=5):
        """
        Want to be able to support multiple nodes on one box (for testing)
        seed_nodes should be the form (address,port,tls) address can be fqdn or ip.
        """
        self.__dict__ = self.cluster_state
        if self.cluster_state != {}:
            return

        self._timeout = timeout

        self.user = user
        self.password = password
        self.auth_mode = auth_mode

        Cluster.use_services_alumni = use_services_alumni
        Cluster.use_services_alt = use_services_alt

        # self.nodes is a dict from Node ID -> Node objects
        self.nodes = {}

        # to avoid multiple entries of endpoints for same server we are keeping
        # this pointers
        self.aliases = {}

        # self.node_lookup is a dict of (fqdn, port) -> Node
        # and (ip, port) -> Node, and node.node_id -> Node
        self.node_lookup = LookupDict()

        self._seed_nodes = set(seed_nodes)
        self._live_nodes = set()
        self.ssl_context = ssl_context

        # crawl the cluster search for nodes in addition to the seed nodes.
        self.last_cluster_refresh_time = 0
        self.only_connect_seed = only_connect_seed
        self._refresh_cluster()

        # to avoid same label (NODE column) for multiple nodes we need to keep track
        # of available nodes name, if names are same then we can use ip:port
        self._same_name_nodes = False

    def __str__(self):
        nodes = self.nodes.values()
        if len(nodes) == 0:
            return ""

        online = [n.key for n in [n for n in nodes if n.alive]]
        offline = [n.key for n in [n for n in nodes if not n.alive]]

        retval = "Found %s nodes" % (len(nodes))
        if online:
            retval += "\nOnline:  %s" % (", ".join(online))
        if offline:
            retval += "\nOffline: %s" % (", ".join(offline))

        return retval

    def get_node_displaynames(self):
        node_names = {}
        for node_key, node in self.nodes.items():
            k = node.sock_name(use_fqdn=True)
            if commonutil.is_valid_ip_port(k):
                node_names[node_key] = k
            else:
                node_names[node_key] = self.node_lookup.get_shortname(k, min_prefix_len=20, min_suffix_len=5)

        return node_names

    def get_node_names(self):
        node_names = {}

        if not self._same_name_nodes:
            for node_key, node in self.nodes.items():
                name = node.sock_name(use_fqdn=True)
                if name in list(node_names.values()):
                    # found same name for multiple nodes
                    self._same_name_nodes = True
                    node_names.clear()
                    break
                node_names[node_key] = name

        if not node_names:
            for node_key, node in self.nodes.items():
                node_names[node_key] = node.sock_name(use_fqdn=False)

        return node_names

    def get_expected_principal(self):
        try:
            principal = "0"
            for k in self.nodes.keys():
                n = self.nodes[k]
                if n.node_id.zfill(16) > principal.zfill(16):
                    principal = n.node_id
            return principal
        except Exception as e:
            self.logger.error(e)
            return ''

    def get_live_nodes(self):
        return self._live_nodes

    def get_visibility_error_nodes(self):
        visible = self.get_live_nodes()
        cluster_visibility_error_nodes = []
        for k in self.nodes.keys():
            node = self.nodes[k]
            if not node.alive:
                # in case of using alumni services, we might have offline nodes
                # which can't detect online nodes
                continue
            peers = util.flatten(node.peers)
            not_visible = set(visible) - set(peers)
{'fraction_non_alphanumeric': 0.07977315689981096, 'fraction_numerical': 0.004536862003780718, 'mean_word_length': 2.1397626112759642, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5485224', 'n_tokens_mistral': 3074, 'n_tokens_neox': 2922, 'n_words': 945}