<commit_before># Copyright (c) 2015, <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import time
import logging

import humanfriendly
import requests.exceptions

try:
    from kazoo.client import KazooClient
except ImportError:
    KazooClient = None

import docker
import docker.errors
import docker.utils

from zoe_master.stats import SwarmStats, SwarmNodeStats
from zoe_lib.exceptions import ZoeException

log = logging.getLogger(__name__)


def zookeeper_swarm(zk_server_list: str, path='/docker'):
    """
    Given a Zookeeper server list, find the currently active Swarm master.
    :param zk_server_list: Zookeeper server list
    :param path: Swarm path in Zookeeper
    :return: Swarm master connection string
    """
    path += '/docker/swarm/leader'
    zk = KazooClient(hosts=zk_server_list)
    zk.start()
    master, stat = zk.get(path)
    zk.stop()
    return master


class SwarmClient:
    def __init__(self, opts):
        self.opts = opts
        url = opts.swarm
        if 'zk://' in url:
            url = url[len('zk://'):]
            manager = zookeeper_swarm(url)
            manager = manager.decode('utf-8')
        elif 'http://' or 'https://' in url:
            manager = url
        else:
            raise ZoeException('Unsupported URL scheme for Swarm')
        log.info('Connecting to Swarm at {}'.format(manager))
        self.cli = docker.Client(base_url=manager)

    def info(self) -> SwarmStats:
        info = self.cli.info()
        pl_status = SwarmStats()
        pl_status.container_count = info["Containers"]
        pl_status.image_count = info["Images"]
        pl_status.memory_total = info["MemTotal"]
        pl_status.cores_total = info["NCPU"]

        # DriverStatus is a list...
        idx = 1
        assert 'Strategy' in info["DriverStatus"][idx][0]
        pl_status.placement_strategy = info["DriverStatus"][idx][1]
        idx = 2
        assert 'Filters' in info["DriverStatus"][idx][0]
        pl_status.active_filters = [x.strip() for x in info["DriverStatus"][idx][1].split(", ")]
        idx = 3
        assert 'Nodes' in info["DriverStatus"][idx][0]
        node_count = int(info["DriverStatus"][idx][1])
        idx = 4
        for node in range(node_count):
            idx2 = 0
            ns = SwarmNodeStats(info["DriverStatus"][idx + node][0])
            ns.docker_endpoint = info["DriverStatus"][idx + node][1]
            idx2 += 1
            if 'Status' in info["DriverStatus"][idx + node + idx2][0]:  # new docker version
                ns.status = info["DriverStatus"][idx + node + idx2][1]
                idx2 += 1
            ns.container_count = int(info["DriverStatus"][idx + node + idx2][1])
            idx2 += 1
            ns.cores_reserved = int(info["DriverStatus"][idx + node + idx2][1].split(' / ')[0])
            ns.cores_total = int(info["DriverStatus"][idx + node + idx2][1].split(' / ')[1])
            idx2 += 1
            ns.memory_reserved = info["DriverStatus"][idx + node + idx2][1].split(' / ')[0]
            ns.memory_total = info["DriverStatus"][idx + node + idx2][1].split(' / ')[1]
            idx2 += 1
            ns.labels = info["DriverStatus"][idx + node + idx2][1:]
            idx2 += 1
            ns.error = info["DriverStatus"][idx + node + idx2][1]
            idx2 += 1
            ns.last_update = info["DriverStatus"][idx + node + idx2][1]

            ns.memory_reserved = humanfriendly.parse_size(ns.memory_reserved)
            ns.memory_total = humanfriendly.parse_size(ns.memory_total)

            pl_status.nodes.append(ns)
            idx += idx2
        pl_status.timestamp = time.time()
        return pl_status

    def spawn_container(self, image, options) -> dict:
        cont = None
        port_bindings = {}
        for port in options.ports:
            port_bindings[port] = None

        if options.gelf_log_address != '':
            log_config = docker.utils.LogConfig(type="gelf", config={'gelf-address': options.gelf_log_address, 'labels': ",".join(options.labels)})
        else:
            log_config = docker.utils.LogConfig(type="json-file")
<commit_msg>Update stats for latest Docker version (will break with older versions)<commit_after># Copyright (c) 2015, <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import time
import logging

import humanfriendly
import requests.exceptions

try:
    from kazoo.client import KazooClient
except ImportError:
    KazooClient = None

import docker
import docker.errors
import docker.utils

from zoe_master.stats import SwarmStats, SwarmNodeStats
from zoe_lib.exceptions import ZoeException

log = logging.getLogger(__name__)


def zookeeper_swarm(zk_server_list: str, path='/docker'):
    """
    Given a Zookeeper server list, find the currently active Swarm master.
    :param zk_server_list: Zookeeper server list
    :param path: Swarm path in Zookeeper
    :return: Swarm master connection string
    """
    path += '/docker/swarm/leader'
    zk = KazooClient(hosts=zk_server_list)
    zk.start()
    master, stat = zk.get(path)
    zk.stop()
    return master


class SwarmClient:
    def __init__(self, opts):
        self.opts = opts
        url = opts.swarm
        if 'zk://' in url:
            url = url[len('zk://'):]
            manager = zookeeper_swarm(url)
            manager = manager.decode('utf-8')
        elif 'http://' or 'https://' in url:
            manager = url
        else:
            raise ZoeException('Unsupported URL scheme for Swarm')
        log.info('Connecting to Swarm at {}'.format(manager))
        self.cli = docker.Client(base_url=manager)

    def info(self) -> SwarmStats:
        info = self.cli.info()
        pl_status = SwarmStats()
        pl_status.container_count = info["Containers"]
        pl_status.image_count = info["Images"]
        pl_status.memory_total = info["MemTotal"]
        pl_status.cores_total = info["NCPU"]

        # DriverStatus is a list...
        idx = 1
        assert 'Strategy' in info["DriverStatus"][idx][0]
        pl_status.placement_strategy = info["DriverStatus"][idx][1]
        idx = 2
        assert 'Filters' in info["DriverStatus"][idx][0]
        pl_status.active_filters = [x.strip() for x in info["DriverStatus"][idx][1].split(", ")]
        idx = 3
        assert 'Nodes' in info["DriverStatus"][idx][0]
        node_count = int(info["DriverStatus"][idx][1])
        idx = 4
        for node in range(node_count):
            idx2 = 0
            ns = SwarmNodeStats(info["DriverStatus"][idx + node][0])
            ns.docker_endpoint = info["DriverStatus"][idx + node][1]
            idx2 += 1
            if 'Status' in info["DriverStatus"][idx + node + idx2][0]:  # new docker version
                ns.status = info["DriverStatus"][idx + node + idx2][1]
                idx2 += 1
            ns.container_count = int(info["DriverStatus"][idx + node + idx2][1])
            idx2 += 1
            ns.cores_reserved = int(info["DriverStatus"][idx + node + idx2][1].split(' / ')[0])
            ns.cores_total = int(info["DriverStatus"][idx + node + idx2][1].split(' / ')[1])
            idx2 += 1
            ns.memory_reserved = info["DriverStatus"][idx + node + idx2][1].split(' / ')[0]
            ns.memory_total = info["DriverStatus"][idx + node + idx2][1].split(' / ')[1]
            idx2 += 1
            ns.labels = info["DriverStatus"][idx + node + idx2][1:]
            idx2 += 1
            ns.error = info["DriverStatus"][idx + node + idx2][1]
            idx2 += 1
            ns.last_update = info["DriverStatus"][idx + node + idx2][1]
            idx2 += 1
            ns.server_version = info["DriverStatus"][idx + node + idx2][1]

            ns.memory_reserved = humanfriendly.parse_size(ns.memory_reserved)
            ns.memory_total = humanfriendly.parse_size(ns.memory_total)

            pl_status.nodes.append(ns)
            idx += idx2
        pl_status.timestamp = time.time()
        return pl_status

    def spawn_container(self, image, options) -> dict:
        cont = None
        port_bindings = {}
        for port in options.ports:
            port_bindings[port] = None

        if options.gelf_log_address != '':
            log_config = docker.utils.LogConfig(type="gelf", config={'gelf-address': options.gelf_log_address, 'labels': ",".join(options.labels)})
        else:
            log_config = docker.utils.LogConfig(type="json-file")

{'fraction_non_alphanumeric': 0.12437440102225535, 'fraction_numerical': 0.013630071344904696, 'mean_word_length': 2.5629742033383915, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3708444', 'n_tokens_mistral': 2876, 'n_tokens_neox': 2774, 'n_words': 736}