<commit_before>import time
import unittest
from datetime import datetime

from docker.models.containers import Container

from seaworthy.checks import docker_client, dockertest
from seaworthy.containers.base import ContainerBase
from seaworthy.dockerhelper import DockerHelper, fetch_images
from seaworthy.logs import EqualsMatcher

IMG_SCRIPT = 'alpine:latest'
IMG_WAIT = 'nginx:alpine'


@dockertest()
def setUpModule():  # noqa: N802 (The camelCase is mandated by unittest.)
    with docker_client() as client:
        fetch_images(client, [IMG_SCRIPT, IMG_WAIT])


@dockertest()
class TestContainerBase(unittest.TestCase):
    def setUp(self):
        self.dh = DockerHelper()
        self.addCleanup(self.dh.teardown)
        self.dh.setup()

        self.base = self.with_cleanup(ContainerBase(
            'wait', IMG_WAIT, docker_helper=self.dh))

    def with_cleanup(self, container):
        self.addCleanup(container._teardown)
        return container

    def test_wait_timeout_default(self):
        """
        When wait_timeout isn't passed to the constructor, the default timeout
        is used.
        """
        container = ContainerBase('timeout', IMG_WAIT)
        self.assertEqual(container.wait_timeout, ContainerBase.WAIT_TIMEOUT)

    def test_wait_timeout_override(self):
        """
        When wait_timeout is passed to the constructor, it is used in place of
        the default.
        """
        timeout = ContainerBase.WAIT_TIMEOUT + 10.0
        container = ContainerBase('timeout', IMG_WAIT, wait_timeout=timeout)
        self.assertEqual(container.wait_timeout, timeout)

    def test_docker_helper_not_set(self):
        """
        By default, we have no docker_helper.
        """
        nodh = ContainerBase('dh', IMG_WAIT)
        self.assertIsNone(nodh._docker_helper)
        with self.assertRaises(RuntimeError) as cm:
            nodh.docker_helper
        self.assertEqual(str(cm.exception), 'No docker_helper set.')

    def test_docker_helper_set_in_constructor(self):
        """
        We can set a docker helper in the constructor.
        """
        withdh = ContainerBase('withdh', IMG_WAIT, docker_helper=self.dh)
        self.assertIs(withdh._docker_helper, self.dh)
        self.assertIs(withdh.docker_helper, self.dh)

    def test_docker_helper_set_to_none(self):
        """
        Setting docker_helper to None has no effect even if we already have
        one.
        """
        nodh = ContainerBase('nodh', IMG_WAIT)
        self.assertIsNone(nodh._docker_helper)
        nodh.set_docker_helper(None)
        self.assertIsNone(nodh._docker_helper)

        withdh = ContainerBase('withdh', IMG_WAIT, docker_helper=self.dh)
        self.assertIs(withdh._docker_helper, self.dh)
        withdh.set_docker_helper(None)
        self.assertIs(withdh._docker_helper, self.dh)

    def test_docker_helper_set_to_current(self):
        """
        Setting docker_helper to the one we already have has no effect.
        """
        withdh = ContainerBase('withdh', IMG_WAIT, docker_helper=self.dh)
        self.assertIs(withdh._docker_helper, self.dh)
        withdh.set_docker_helper(self.dh)
        self.assertIs(withdh._docker_helper, self.dh)

    def test_cannot_replace_docker_helper(self):
        """
        If we already have a docker_helper, we can't set a different one.
        """
        withdh = ContainerBase('withdh', IMG_WAIT, docker_helper=self.dh)
        self.assertIs(withdh.docker_helper, self.dh)
        with self.assertRaises(RuntimeError) as cm:
            withdh.set_docker_helper(DockerHelper())
        self.assertEqual(
            str(cm.exception), 'Cannot replace existing docker_helper.')
        self.assertIs(withdh.docker_helper, self.dh)

    def test_create_only_if_not_created(self):
        """The container cannot be created more than once."""
        self.base.create_and_start(pull=False)

        # We can't create the container when it's already created
        with self.assertRaises(RuntimeError) as cm:
            self.base.create_and_start(pull=False)
        self.assertEqual(str(cm.exception), 'Container already created.')

        self.base.stop_and_remove()

    def test_remove_only_if_created(self):
        """The container can only be removed if it has been created."""
        self.base.create_and_start(pull=False)

        # We can remove the container if it's created
        self.base.stop_and_remove()

        with self.assertRaises(RuntimeError) as cm:
            self.base.stop_and_remove()
        self.assertEqual(str(cm.exception), 'Container not created yet.')

    def test_container_only_if_created(self):
        """
        We can only access the inner Container object if the container has been
        created.
        """
        # If we try get the container before it's created it'll fail
        with self.assertRaises(RuntimeError) as cm:
            self.base.inner()
        self.assertEqual(str(cm.exception), 'Container not created yet.')

        self.base.create_and_start(pull=False)

        # We can get the container once it's created
        container = self.base.inner()
        self.assertIsInstance(container, Container)

        self.base.stop_and_remove()
        with self.assertRaises(RuntimeError) as cm:
            self.base.inner()
        self.assertEqual(str(cm.exception), 'Container not created yet.')

    def test_merge_kwargs(self):
        """
        The default merge_kwargs() method deep-merges the two kwargs dicts
        passed to it.
        """
        create_kwargs = {'a': {'aa': 1, 'ab': 2}, 's': 'foo', 't': 'bar'}
        kwargs = {'a': {'ba': 3, 'ab': 4}, 'r': 'arr', 't': 'baz'}
        self.assertEqual(self.base.merge_kwargs(create_kwargs, kwargs), {
            'a': {'aa': 1, 'ab': 4, 'ba': 3},
            'r': 'arr',
            's': 'foo',
            't': 'baz',
        })

    def test_merge_kwargs_dicts_only(self):
        """
        The kwargs we merge must be dicts.
        """
        with self.assertRaises(Exception):
            self.base.merge_kwargs({}, 'hello')
        with self.assertRaises(Exception):
            self.base.merge_kwargs('hello', {})

    def test_create_kwargs_handling(self):
        """
        The keyword args passed used for container creation come from the
        return value of merge_kwargs() called on create_kwargs from the
        constructor and kwargs from the create_and_start() method.
        """
        create_kwargs = {
            'environment': {'CREATE_KWARGS': 't', 'KWARGS_MERGED': 'f'},
        }
        kwargs = {'environment': {'KWARGS': 't', 'KWARGS_MERGED': 't'}}
        merge_kwargs_args = []

        class SubContainer(ContainerBase):
            def merge_kwargs(self, *args):
                merge_kwargs_args.extend(args)
                return super().merge_kwargs(*args)

        c = self.with_cleanup(SubContainer(
            'kwargs', IMG_WAIT, create_kwargs=create_kwargs))
        c.create_and_start(self.dh, pull=False, kwargs=kwargs)

        self.assertEqual(merge_kwargs_args, [create_kwargs, kwargs])
        c_env = [v for v in c.inner().attrs['Config']['Env'] if 'KWARGS' in v]
        self.assertEqual(
            sorted(c_env), ['CREATE_KWARGS=t', 'KWARGS=t', 'KWARGS_MERGED=t'])

    def test_default_clean(self):
        """By default, the ``clean`` method raises a NotImplementedError."""
        with self.assertRaises(NotImplementedError):
            self.base.clean()

    def test_ports(self):
        """
        We can get the ports exposed or published on a container.
        """
        self.base.create_and_start(self.dh, pull=False, kwargs={'ports': {
            '8000/tcp': ('127.0.0.1', '10701'),
        }})

        # We're not interested in the order of the ports
        self.assertCountEqual(self.base.ports.items(), [
            ('80/tcp', None),
            ('8000/tcp', [{'HostIp': '127.0.0.1', 'HostPort': '10701'}]),
        ])

    def test_get_host_port(self):
        """
        We can get the host port mapping of a container.
        """
        self.base.create_and_start(pull=False, kwargs={'ports': {
            '8080/tcp': ('127.0.0.1',),
            '9090/tcp': ('127.0.0.1', '10701'),
            '9191/udp': '10702',
        }})

        # We get a random high port number here.
        host_iface, random_host_port = self.base.get_host_port('8080')
        self.assertEqual(host_iface, '127.0.0.1')
        self.assertGreater(int(random_host_port), 1024)
        self.assertLess(int(random_host_port), 65536)

        # We get the specific port we defined here.
        host_iface, specific_host_port = self.base.get_host_port('9090')
        self.assertEqual(host_iface, '127.0.0.1')
        self.assertEqual(specific_host_port, '10701')

        # We get a UDP port we defined.
        _, udp_host_port = self.base.get_host_port('9191', proto='udp')
        self.assertEqual(udp_host_port, '10702')

        # FIXME: Don't bother testing index != 0, the port order is
        # unpredictable :-(

    def test_get_host_port_not_exposed(self):
        """
        When we try to get the host port for a container port that hasn't been
        exposed, an error is raised.
        """
        self.base.create_and_start(self.dh, pull=False, kwargs={'ports': {
            '8000/tcp': ('127.0.0.1', '10701'),
        }})

        self.assertNotIn('90/tcp', self.base.ports)

        with self.assertRaises(ValueError) as cm:
            self.base.get_host_port('90')
        self.assertEqual(str(cm.exception), "Port '90/tcp' is not exposed")

    def test_get_host_port_not_published(self):
        """
        When we try to get the host port for a container port that hasn't been
        published to the host, an error is raised.
        """
        self.base.create_and_start(self.dh, pull=False, kwargs={'ports': {
            '8000/tcp': ('127.0.0.1', '10701'),
        }})

        # The Nginx image EXPOSEs port 80, but we don't publish it
        self.assertIn('80/tcp', self.base.ports)

        with self.assertRaises(ValueError) as cm:
            self.base.get_host_port('80')
        self.assertEqual(
            str(cm.exception), "Port '80/tcp' is not published to the host")

    def test_get_first_host_port(self):
        """
        When we get the first host port for the container, the host port mapped
        to the lowest container port is returned.
        """
        self.base.create_and_start(self.dh, pull=False, kwargs={'ports': {
            '8000/tcp': ('127.0.0.1',),
            '90/tcp': ('127.0.0.1', '10701'),
            '90/udp': ('127.0.0.1', '10702'),
        }})

        # The Nginx image EXPOSEs port 80, but it's not published so shouldn't
        # be considered by ``get_first_host_port()``
        self.assertIn('80/tcp', self.base.ports)

        host_iface, host_port = self.base.get_first_host_port()
        self.assertEqual(host_iface, '127.0.0.1')
        self.assertEqual(host_port, '10701')

    def test_get_first_host_port_no_mappings(self):
        """
        When we try to get the first host port, but the container has no
        published ports, an error is raised.
        """
        self.base.create_and_start(self.dh, pull=False)

        # The Nginx image EXPOSEs port 80, but it's not published so shouldn't
        # be considered by ``get_first_host_port()``
        self.assertIn('80/tcp', self.base.ports)

        with self.assertRaises(RuntimeError) as cm:
            self.base.get_first_host_port()

        self.assertEqual(str(cm.exception), 'Container has no published ports')

    def run_logs_container(self, logs, wait=True, delay=0.01):
        # Sleep some amount between lines to ensure ordering across stdout and
        # stderr.
        script = '\nsleep {}\n'.format(delay).join(logs)

        script_con = self.with_cleanup(ContainerBase('script', IMG_SCRIPT))

        script_con.create_and_start(self.dh, pull=False, kwargs={
            'command': ['sh', '-c', script],
        })
        # Wait for the output to arrive.
        if wait:
            # Wait a minimum of 100ms to avoid jitter with small intervals.
            time.sleep(max(0.1, len(logs) * delay))
        return script_con

    def test_get_logs_out_err(self):
        """
        We can choose stdout and/or stderr when getting logs from a container.
        """
        script = self.run_logs_container([
            'echo "o0"', 'echo "e0" >&2',
            'echo "o1"', 'echo "e1" >&2',
        ])

        self.assertEqual(script.get_logs(), b'o0\ne0\no1\ne1\n')
        self.assertEqual(script.get_logs(stdout=False), b'e0\ne1\n')
        self.assertEqual(script.get_logs(stderr=False), b'o0\no1\n')

    def test_get_logs_tail(self):
        """
        We can choose how many lines to tail when getting logs from a
        container.

        NOTE: Lines are tailed *before* stdout/stderr are filtered out.
        """
        script = self.run_logs_container([
            'echo "o0"', 'echo "e0" >&2',
            'echo "o1"', 'echo "e1" >&2',
        ])

        self.assertEqual(script.get_logs(tail='all'), b'o0\ne0\no1\ne1\n')
        self.assertEqual(script.get_logs(tail=10000), b'o0\ne0\no1\ne1\n')
        self.assertEqual(script.get_logs(tail=0), b'')
        self.assertEqual(script.get_logs(tail=1), b'e1\n')
        self.assertEqual(script.get_logs(tail=2), b'o1\ne1\n')
        # The entries are tailed *before* they're filtered. :-(
        self.assertEqual(script.get_logs(stderr=False, tail=1), b'')
        self.assertEqual(script.get_logs(stderr=False, tail=2), b'o1\n')
        self.assertEqual(script.get_logs(stderr=False, tail=3), b'o1\n')
        self.assertEqual(script.get_logs(stderr=False, tail=4), b'o0\no1\n')

    def test_get_logs_timestamps(self):
        """
        We can ask for timestamps on our logs.
        """
        before = datetime.utcnow()
        script = self.run_logs_container([
            'echo "o0"', 'echo "e0" >&2',
            'echo "o1"', 'echo "e1" >&2',
        ])
        after = datetime.utcnow()

        self.assertEqual(script.get_logs(), b'o0\ne0\no1\ne1\n')
        raw_lines = script.get_logs(timestamps=True).splitlines()

        earlier = before
        for line, expected in zip(raw_lines, [b'o0', b'e0', b'o1', b'e1']):
            ts, l = line.split(b' ', 1)
            self.assertEqual(l, expected)
            # Truncate the nanoseconds, because we can't parse them.
            ts = (ts[:26] + ts[-1:]).decode('utf8')
            ts = datetime.strptime(ts, '%Y-%m-%dT%H:%M:%S.%fZ')
            self.assertLess(earlier, ts)
            earlier = ts
        self.assertLess(earlier, after)

    def test_stream_logs_timeout(self):
        """
        We can stream logs with a timeout.
        """
        script = self.run_logs_container([
            'echo "o0"', 'echo "e0" >&2',
            'echo "o1"', 'echo "e1" >&2',
        ], delay=0.25, wait=False)

        time.sleep(0.01)
        lines = []
        with self.assertRaises(TimeoutError):
            for line in script.stream_logs(timeout=0.3):
                lines.append(line)<commit_msg>New pep8, new style check. (@JayH5)<commit_after>import time
import unittest
from datetime import datetime

from docker.models.containers import Container

from seaworthy.checks import docker_client, dockertest
from seaworthy.containers.base import ContainerBase
from seaworthy.dockerhelper import DockerHelper, fetch_images
from seaworthy.logs import EqualsMatcher

IMG_SCRIPT = 'alpine:latest'
IMG_WAIT = 'nginx:alpine'


@dockertest()
def setUpModule():  # noqa: N802 (The camelCase is mandated by unittest.)
    with docker_client() as client:
        fetch_images(client, [IMG_SCRIPT, IMG_WAIT])


@dockertest()
class TestContainerBase(unittest.TestCase):
    def setUp(self):
        self.dh = DockerHelper()
        self.addCleanup(self.dh.teardown)
        self.dh.setup()

        self.base = self.with_cleanup(ContainerBase(
            'wait', IMG_WAIT, docker_helper=self.dh))

    def with_cleanup(self, container):
        self.addCleanup(container._teardown)
        return container

    def test_wait_timeout_default(self):
        """
        When wait_timeout isn't passed to the constructor, the default timeout
        is used.
        """
        container = ContainerBase('timeout', IMG_WAIT)
        self.assertEqual(container.wait_timeout, ContainerBase.WAIT_TIMEOUT)

    def test_wait_timeout_override(self):
        """
        When wait_timeout is passed to the constructor, it is used in place of
        the default.
        """
        timeout = ContainerBase.WAIT_TIMEOUT + 10.0
        container = ContainerBase('timeout', IMG_WAIT, wait_timeout=timeout)
        self.assertEqual(container.wait_timeout, timeout)

    def test_docker_helper_not_set(self):
        """
        By default, we have no docker_helper.
        """
        nodh = ContainerBase('dh', IMG_WAIT)
        self.assertIsNone(nodh._docker_helper)
        with self.assertRaises(RuntimeError) as cm:
            nodh.docker_helper
        self.assertEqual(str(cm.exception), 'No docker_helper set.')

    def test_docker_helper_set_in_constructor(self):
        """
        We can set a docker helper in the constructor.
        """
        withdh = ContainerBase('withdh', IMG_WAIT, docker_helper=self.dh)
        self.assertIs(withdh._docker_helper, self.dh)
        self.assertIs(withdh.docker_helper, self.dh)

    def test_docker_helper_set_to_none(self):
        """
        Setting docker_helper to None has no effect even if we already have
        one.
        """
        nodh = ContainerBase('nodh', IMG_WAIT)
        self.assertIsNone(nodh._docker_helper)
        nodh.set_docker_helper(None)
        self.assertIsNone(nodh._docker_helper)

        withdh = ContainerBase('withdh', IMG_WAIT, docker_helper=self.dh)
        self.assertIs(withdh._docker_helper, self.dh)
        withdh.set_docker_helper(None)
        self.assertIs(withdh._docker_helper, self.dh)

    def test_docker_helper_set_to_current(self):
        """
        Setting docker_helper to the one we already have has no effect.
        """
        withdh = ContainerBase('withdh', IMG_WAIT, docker_helper=self.dh)
        self.assertIs(withdh._docker_helper, self.dh)
        withdh.set_docker_helper(self.dh)
        self.assertIs(withdh._docker_helper, self.dh)

    def test_cannot_replace_docker_helper(self):
        """
        If we already have a docker_helper, we can't set a different one.
        """
        withdh = ContainerBase('withdh', IMG_WAIT, docker_helper=self.dh)
        self.assertIs(withdh.docker_helper, self.dh)
        with self.assertRaises(RuntimeError) as cm:
            withdh.set_docker_helper(DockerHelper())
        self.assertEqual(
            str(cm.exception), 'Cannot replace existing docker_helper.')
        self.assertIs(withdh.docker_helper, self.dh)

    def test_create_only_if_not_created(self):
        """The container cannot be created more than once."""
        self.base.create_and_start(pull=False)

        # We can't create the container when it's already created
        with self.assertRaises(RuntimeError) as cm:
            self.base.create_and_start(pull=False)
        self.assertEqual(str(cm.exception), 'Container already created.')

        self.base.stop_and_remove()

    def test_remove_only_if_created(self):
        """The container can only be removed if it has been created."""
        self.base.create_and_start(pull=False)

        # We can remove the container if it's created
        self.base.stop_and_remove()

        with self.assertRaises(RuntimeError) as cm:
            self.base.stop_and_remove()
        self.assertEqual(str(cm.exception), 'Container not created yet.')

    def test_container_only_if_created(self):
        """
        We can only access the inner Container object if the container has been
        created.
        """
        # If we try get the container before it's created it'll fail
        with self.assertRaises(RuntimeError) as cm:
            self.base.inner()
        self.assertEqual(str(cm.exception), 'Container not created yet.')

        self.base.create_and_start(pull=False)

        # We can get the container once it's created
        container = self.base.inner()
        self.assertIsInstance(container, Container)

        self.base.stop_and_remove()
        with self.assertRaises(RuntimeError) as cm:
            self.base.inner()
        self.assertEqual(str(cm.exception), 'Container not created yet.')

    def test_merge_kwargs(self):
        """
        The default merge_kwargs() method deep-merges the two kwargs dicts
        passed to it.
        """
        create_kwargs = {'a': {'aa': 1, 'ab': 2}, 's': 'foo', 't': 'bar'}
        kwargs = {'a': {'ba': 3, 'ab': 4}, 'r': 'arr', 't': 'baz'}
        self.assertEqual(self.base.merge_kwargs(create_kwargs, kwargs), {
            'a': {'aa': 1, 'ab': 4, 'ba': 3},
            'r': 'arr',
            's': 'foo',
            't': 'baz',
        })

    def test_merge_kwargs_dicts_only(self):
        """
        The kwargs we merge must be dicts.
        """
        with self.assertRaises(Exception):
            self.base.merge_kwargs({}, 'hello')
        with self.assertRaises(Exception):
            self.base.merge_kwargs('hello', {})

    def test_create_kwargs_handling(self):
        """
        The keyword args passed used for container creation come from the
        return value of merge_kwargs() called on create_kwargs from the
        constructor and kwargs from the create_and_start() method.
        """
        create_kwargs = {
            'environment': {'CREATE_KWARGS': 't', 'KWARGS_MERGED': 'f'},
        }
        kwargs = {'environment': {'KWARGS': 't', 'KWARGS_MERGED': 't'}}
        merge_kwargs_args = []

        class SubContainer(ContainerBase):
            def merge_kwargs(self, *args):
                merge_kwargs_args.extend(args)
                return super().merge_kwargs(*args)

        c = self.with_cleanup(SubContainer(
            'kwargs', IMG_WAIT, create_kwargs=create_kwargs))
        c.create_and_start(self.dh, pull=False, kwargs=kwargs)

        self.assertEqual(merge_kwargs_args, [create_kwargs, kwargs])
        c_env = [v for v in c.inner().attrs['Config']['Env'] if 'KWARGS' in v]
        self.assertEqual(
            sorted(c_env), ['CREATE_KWARGS=t', 'KWARGS=t', 'KWARGS_MERGED=t'])

    def test_default_clean(self):
        """By default, the ``clean`` method raises a NotImplementedError."""
        with self.assertRaises(NotImplementedError):
            self.base.clean()

    def test_ports(self):
        """
        We can get the ports exposed or published on a container.
        """
        self.base.create_and_start(self.dh, pull=False, kwargs={'ports': {
            '8000/tcp': ('127.0.0.1', '10701'),
        }})

        # We're not interested in the order of the ports
        self.assertCountEqual(self.base.ports.items(), [
            ('80/tcp', None),
            ('8000/tcp', [{'HostIp': '127.0.0.1', 'HostPort': '10701'}]),
        ])

    def test_get_host_port(self):
        """
        We can get the host port mapping of a container.
        """
        self.base.create_and_start(pull=False, kwargs={'ports': {
            '8080/tcp': ('127.0.0.1',),
            '9090/tcp': ('127.0.0.1', '10701'),
            '9191/udp': '10702',
        }})

        # We get a random high port number here.
        host_iface, random_host_port = self.base.get_host_port('8080')
        self.assertEqual(host_iface, '127.0.0.1')
        self.assertGreater(int(random_host_port), 1024)
        self.assertLess(int(random_host_port), 65536)

        # We get the specific port we defined here.
        host_iface, specific_host_port = self.base.get_host_port('9090')
        self.assertEqual(host_iface, '127.0.0.1')
        self.assertEqual(specific_host_port, '10701')

        # We get a UDP port we defined.
        _, udp_host_port = self.base.get_host_port('9191', proto='udp')
        self.assertEqual(udp_host_port, '10702')

        # FIXME: Don't bother testing index != 0, the port order is
        # unpredictable :-(

    def test_get_host_port_not_exposed(self):
        """
        When we try to get the host port for a container port that hasn't been
        exposed, an error is raised.
        """
        self.base.create_and_start(self.dh, pull=False, kwargs={'ports': {
            '8000/tcp': ('127.0.0.1', '10701'),
        }})

        self.assertNotIn('90/tcp', self.base.ports)

        with self.assertRaises(ValueError) as cm:
            self.base.get_host_port('90')
        self.assertEqual(str(cm.exception), "Port '90/tcp' is not exposed")

    def test_get_host_port_not_published(self):
        """
        When we try to get the host port for a container port that hasn't been
        published to the host, an error is raised.
        """
        self.base.create_and_start(self.dh, pull=False, kwargs={'ports': {
            '8000/tcp': ('127.0.0.1', '10701'),
        }})

        # The Nginx image EXPOSEs port 80, but we don't publish it
        self.assertIn('80/tcp', self.base.ports)

        with self.assertRaises(ValueError) as cm:
            self.base.get_host_port('80')
        self.assertEqual(
            str(cm.exception), "Port '80/tcp' is not published to the host")

    def test_get_first_host_port(self):
        """
        When we get the first host port for the container, the host port mapped
        to the lowest container port is returned.
        """
        self.base.create_and_start(self.dh, pull=False, kwargs={'ports': {
            '8000/tcp': ('127.0.0.1',),
            '90/tcp': ('127.0.0.1', '10701'),
            '90/udp': ('127.0.0.1', '10702'),
        }})

        # The Nginx image EXPOSEs port 80, but it's not published so shouldn't
        # be considered by ``get_first_host_port()``
        self.assertIn('80/tcp', self.base.ports)

        host_iface, host_port = self.base.get_first_host_port()
        self.assertEqual(host_iface, '127.0.0.1')
        self.assertEqual(host_port, '10701')

    def test_get_first_host_port_no_mappings(self):
        """
        When we try to get the first host port, but the container has no
        published ports, an error is raised.
        """
        self.base.create_and_start(self.dh, pull=False)

        # The Nginx image EXPOSEs port 80, but it's not published so shouldn't
        # be considered by ``get_first_host_port()``
        self.assertIn('80/tcp', self.base.ports)

        with self.assertRaises(RuntimeError) as cm:
            self.base.get_first_host_port()

        self.assertEqual(str(cm.exception), 'Container has no published ports')

    def run_logs_container(self, logs, wait=True, delay=0.01):
        # Sleep some amount between lines to ensure ordering across stdout and
        # stderr.
        script = '\nsleep {}\n'.format(delay).join(logs)

        script_con = self.with_cleanup(ContainerBase('script', IMG_SCRIPT))

        script_con.create_and_start(self.dh, pull=False, kwargs={
            'command': ['sh', '-c', script],
        })
        # Wait for the output to arrive.
        if wait:
            # Wait a minimum of 100ms to avoid jitter with small intervals.
            time.sleep(max(0.1, len(logs) * delay))
        return script_con

    def test_get_logs_out_err(self):
        """
        We can choose stdout and/or stderr when getting logs from a container.
        """
        script = self.run_logs_container([
            'echo "o0"', 'echo "e0" >&2',
            'echo "o1"', 'echo "e1" >&2',
        ])

        self.assertEqual(script.get_logs(), b'o0\ne0\no1\ne1\n')
        self.assertEqual(script.get_logs(stdout=False), b'e0\ne1\n')
        self.assertEqual(script.get_logs(stderr=False), b'o0\no1\n')

    def test_get_logs_tail(self):
        """
        We can choose how many lines to tail when getting logs from a
        container.

        NOTE: Lines are tailed *before* stdout/stderr are filtered out.
        """
        script = self.run_logs_container([
            'echo "o0"', 'echo "e0" >&2',
            'echo "o1"', 'echo "e1" >&2',
        ])

        self.assertEqual(script.get_logs(tail='all'), b'o0\ne0\no1\ne1\n')
        self.assertEqual(script.get_logs(tail=10000), b'o0\ne0\no1\ne1\n')
        self.assertEqual(script.get_logs(tail=0), b'')
        self.assertEqual(script.get_logs(tail=1), b'e1\n')
        self.assertEqual(script.get_logs(tail=2), b'o1\ne1\n')
        # The entries are tailed *before* they're filtered. :-(
        self.assertEqual(script.get_logs(stderr=False, tail=1), b'')
        self.assertEqual(script.get_logs(stderr=False, tail=2), b'o1\n')
        self.assertEqual(script.get_logs(stderr=False, tail=3), b'o1\n')
        self.assertEqual(script.get_logs(stderr=False, tail=4), b'o0\no1\n')

    def test_get_logs_timestamps(self):
        """
        We can ask for timestamps on our logs.
        """
        before = datetime.utcnow()
        script = self.run_logs_container([
            'echo "o0"', 'echo "e0" >&2',
            'echo "o1"', 'echo "e1" >&2',
        ])
        after = datetime.utcnow()

        self.assertEqual(script.get_logs(), b'o0\ne0\no1\ne1\n')
        raw_lines = script.get_logs(timestamps=True).splitlines()

        earlier = before
        for line, expected in zip(raw_lines, [b'o0', b'e0', b'o1', b'e1']):
            ts, ln = line.split(b' ', 1)
            self.assertEqual(ln, expected)
            # Truncate the nanoseconds, because we can't parse them.
            ts = (ts[:26] + ts[-1:]).decode('utf8')
            ts = datetime.strptime(ts, '%Y-%m-%dT%H:%M:%S.%fZ')
            self.assertLess(earlier, ts)
            earlier = ts
        self.assertLess(earlier, after)

    def test_stream_logs_timeout(self):
        """
        We can stream logs with a timeout.
        """
        script = self.run_logs_container([
            'echo "o0"', 'echo "e0" >&2',
            'echo "o1"', 'echo "e1" >&2',
        ], delay=0.25, wait=False)

        time.sleep(0.01)
        lines = []
        with self.assertRaises(TimeoutError):
            for line in script.stream_logs(timeout=0.3):
                lines.append(line)
{'fraction_non_alphanumeric': 0.12397757255936676, 'fraction_numerical': 0.020514511873350922, 'mean_word_length': 2.4609062892363887, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 24, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1074925', 'n_tokens_mistral': 9854, 'n_tokens_neox': 9303, 'n_words': 2198}