<commit_before># Copyright (C) 2013-2014 SignalFuse, Inc.
#
# Docker container orchestration utility.

import bgtunnel
import datetime

# Import _strptime manually to work around a thread safety issue when using
# strptime() from threads for the first time.
import _strptime # flake8: noqa

import docker
try:
    from docker.errors import APIError
except ImportError:
    # Fall back to <= 0.3.1 location
    from docker.client import APIError

import multiprocessing.pool
import re
import six

# For Python bug workaround
import threading
import weakref

from . import exceptions
from . import lifecycle


class Entity:
    """Base class for named entities in the orchestrator."""
    def __init__(self, name):
        self._name = name

    @property
    def name(self):
        """Get the name of this entity."""
        return self._name

    def __repr__(self):
        return self._name


class Ship(Entity):
    """A Ship that can host and run Containers.

    Ships are hosts in the infrastructure. A Docker daemon is expected to be
    running on each ship, providing control over the containers that will be
    executed there.
    """

    DEFAULT_DOCKER_PORT = 4243
    DEFAULT_DOCKER_VERSION = '1.10'
    DEFAULT_DOCKER_TIMEOUT = 5

    def __init__(self, name, ip, endpoint=None, docker_port=None, timeout=None,
                 ssh_tunnel=None):
        """Instantiate a new ship.

        Args:
            name (string): the name of the ship.
            ip (string): the IP address of resolvable host name of the host.
            docker_port (int): the port the Docker daemon listens on.
            ssh_tunnel (dict): configuration for SSH tunneling to the remote
                Docker daemon.
        """
        Entity.__init__(self, name)
        self._ip = ip
        self._endpoint = endpoint or ip
        self._docker_port = int(docker_port or self.DEFAULT_DOCKER_PORT)
        self._tunnel = None

        if ssh_tunnel:
            if 'user' not in ssh_tunnel:
                raise exceptions.EnvironmentConfigurationException(
                    'Missing SSH user for ship {} tunnel configuration'.format(
                        self.name))
            if 'key' not in ssh_tunnel:
                raise exceptions.EnvironmentConfigurationException(
                    'Missing SSH key for ship {} tunnel configuration'.format(
                        self.name))

            self._tunnel = bgtunnel.open(
                ssh_address=self._endpoint,
                ssh_user=ssh_tunnel['user'],
                ssh_port=int(ssh_tunnel.get('port', 22)),
                host_port=self._docker_port,
                silent=True,
                identity_file=ssh_tunnel['key'])
            self._backend_url = 'http://localhost:{}'.format(
                self._tunnel.bind_port)
        else:
            self._backend_url = 'http://{:s}:{:d}'.format(
                self._endpoint, self._docker_port)

        self._backend = docker.Client(
            base_url=self._backend_url,
            version=Ship.DEFAULT_DOCKER_VERSION,
            timeout=timeout or Ship.DEFAULT_DOCKER_TIMEOUT)

    @property
    def ip(self):
        """Returns this ship's IP address or hostname."""
        return self._ip

    @property
    def endpoint(self):
        """Returns this ship's Docker endpoint IP address or hostname."""
        return self._endpoint

    @property
    def backend(self):
        """Returns the Docker client wrapper to talk to the Docker daemon on
        this host."""
        return self._backend

    @property
    def address(self):
        if self._tunnel:
            return '{} (ssh:{})'.format(self.name, self._tunnel.bind_port)
        return self.name
<commit_msg>added ugly wait period for bgtunnel to finish whatever it needs to do<commit_after># Copyright (C) 2013-2014 SignalFuse, Inc.
#
# Docker container orchestration utility.

import bgtunnel
import datetime
import time

# Import _strptime manually to work around a thread safety issue when using
# strptime() from threads for the first time.
import _strptime # flake8: noqa

import docker
try:
    from docker.errors import APIError
except ImportError:
    # Fall back to <= 0.3.1 location
    from docker.client import APIError

import multiprocessing.pool
import re
import six

# For Python bug workaround
import threading
import weakref

from . import exceptions
from . import lifecycle


class Entity:
    """Base class for named entities in the orchestrator."""
    def __init__(self, name):
        self._name = name

    @property
    def name(self):
        """Get the name of this entity."""
        return self._name

    def __repr__(self):
        return self._name


class Ship(Entity):
    """A Ship that can host and run Containers.

    Ships are hosts in the infrastructure. A Docker daemon is expected to be
    running on each ship, providing control over the containers that will be
    executed there.
    """

    DEFAULT_DOCKER_PORT = 4243
    DEFAULT_DOCKER_VERSION = '1.10'
    DEFAULT_DOCKER_TIMEOUT = 5

    def __init__(self, name, ip, endpoint=None, docker_port=None, timeout=None,
                 ssh_tunnel=None):
        """Instantiate a new ship.

        Args:
            name (string): the name of the ship.
            ip (string): the IP address of resolvable host name of the host.
            docker_port (int): the port the Docker daemon listens on.
            ssh_tunnel (dict): configuration for SSH tunneling to the remote
                Docker daemon.
        """
        Entity.__init__(self, name)
        self._ip = ip
        self._endpoint = endpoint or ip
        self._docker_port = int(docker_port or self.DEFAULT_DOCKER_PORT)
        self._tunnel = None

        if ssh_tunnel:
            if 'user' not in ssh_tunnel:
                raise exceptions.EnvironmentConfigurationException(
                    'Missing SSH user for ship {} tunnel configuration'.format(
                        self.name))
            if 'key' not in ssh_tunnel:
                raise exceptions.EnvironmentConfigurationException(
                    'Missing SSH key for ship {} tunnel configuration'.format(
                        self.name))

            self._tunnel = bgtunnel.open(
                ssh_address=self._endpoint,
                ssh_user=ssh_tunnel['user'],
                ssh_port=int(ssh_tunnel.get('port', 22)),
                host_port=self._docker_port,
                silent=True,
                identity_file=ssh_tunnel['key'])
            self._backend_url = 'http://localhost:{}'.format(
                self._tunnel.bind_port)
            
            # Apparently bgtunnel isn't always ready right away and this 
            # drastically cuts down on the timeouts
            time.sleep(1)

        else:
            self._backend_url = 'http://{:s}:{:d}'.format(
                self._endpoint, self._docker_port)

        self._backend = docker.Client(
            base_url=self._backend_url,
            version=Ship.DEFAULT_DOCKER_VERSION,
            timeout=timeout or Ship.DEFAULT_DOCKER_TIMEOUT)

    @property
    def ip(self):
        """Returns this ship's IP address or hostname."""
        return self._ip

    @property
    def endpoint(self):
        """Returns this ship's Docker endpoint IP address or hostname."""
        return self._endpoint

    @property
    def backend(self):
        """Returns the Docker client wrapper to talk to the Docker daemon on
        this host."""
        return self._backend

    @property
    def address(self):
        if self._tunnel:
            return '{} (ssh:{})'.format(self.name, self._tunnel.bind_port)
        return self.name

{'fraction_non_alphanumeric': 0.08040593286494926, 'fraction_numerical': 0.00585480093676815, 'mean_word_length': 2.1568788501026694, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7015629', 'n_tokens_mistral': 2173, 'n_tokens_neox': 2010, 'n_words': 680}