<commit_before>"""
This module is necessary since the docker python library does not yet provide built-in support
for nvidia-docker tooling. If some incarnation of this feature[1] makes its way into a release
of docker-py, this module can go away. Until then, the ideas in that PR have been copied and
lightly modified here.

[1] https://github.com/docker/docker-py/pull/1560
"""

import os
import re
import requests

from docker.api.client import APIClient
from docker.client import DockerClient
from docker.errors import DockerException
from docker.utils.utils import parse_devices

NVIDIA_DEFAULT_HOST = 'localhost'
NVIDIA_DEFAULT_PORT = 3476


def get_nvidia_docker_endpoint():
    host = os.environ.get('NV_HOST', 'http://%s:%s' % (NVIDIA_DEFAULT_HOST, NVIDIA_DEFAULT_PORT))
    return host + '/docker/cli/json'


def get_nvidia_configuration():
    url = get_nvidia_docker_endpoint()
    try:
        return requests.get(url).json()
    except requests.exceptions.ConnectionError:
        url = get_nvidia_docker_endpoint()
        raise NvidiaConnectionError((
            'Couldn\'t connect to nvidia-driver-plugin at {url} - is it running and accessible?.\n'
            'Try: "curl {url}" or "systemctl start nvidia-docker"').format(url=url), url)


def is_nvidia_image(api, image):
    return (api.inspect_image(image).get('Config', {}).get('Labels', {}).
            get('com.nvidia.volumes.needed') == 'nvidia_driver')


def add_nvidia_docker_to_config(container_config):
    if not container_config.get('HostConfig', None):
        container_config['HostConfig'] = {}

    nvidia_config = get_nvidia_configuration()

    # Setup the Volumes
    container_config['HostConfig'].setdefault('VolumeDriver', nvidia_config['VolumeDriver'])
    container_config['HostConfig'].setdefault('Binds', [])
    container_config['HostConfig']['Binds'].extend(nvidia_config['Volumes'])

    # Get nvidia control devices
    devices = container_config['HostConfig'].get('Devices', [])
    # suport both '0 1' and '0, 1' formats, just like nvidia-docker
    gpu_isolation = os.getenv('NV_GPU', '').replace(',', ' ').split()
    pattern = re.compile(r'/nvidia([0-9]+)$')
    for device in nvidia_config['Devices']:
        if gpu_isolation:<commit_msg>Fix pulling images with "labels" property equal to None<commit_after>"""
This module is necessary since the docker python library does not yet provide built-in support
for nvidia-docker tooling. If some incarnation of this feature[1] makes its way into a release
of docker-py, this module can go away. Until then, the ideas in that PR have been copied and
lightly modified here.

[1] https://github.com/docker/docker-py/pull/1560
"""

import os
import re
import requests

from docker.api.client import APIClient
from docker.client import DockerClient
from docker.errors import DockerException
from docker.utils.utils import parse_devices

NVIDIA_DEFAULT_HOST = 'localhost'
NVIDIA_DEFAULT_PORT = 3476


def get_nvidia_docker_endpoint():
    host = os.environ.get('NV_HOST', 'http://%s:%s' % (NVIDIA_DEFAULT_HOST, NVIDIA_DEFAULT_PORT))
    return host + '/docker/cli/json'


def get_nvidia_configuration():
    url = get_nvidia_docker_endpoint()
    try:
        return requests.get(url).json()
    except requests.exceptions.ConnectionError:
        url = get_nvidia_docker_endpoint()
        raise NvidiaConnectionError((
            'Couldn\'t connect to nvidia-driver-plugin at {url} - is it running and accessible?.\n'
            'Try: "curl {url}" or "systemctl start nvidia-docker"').format(url=url), url)


def is_nvidia_image(api, image):
    labels = api.inspect_image(image).get('Config', {}).get('Labels')
    return labels and labels.get('com.nvidia.volumes.needed') == 'nvidia_driver'


def add_nvidia_docker_to_config(container_config):
    if not container_config.get('HostConfig', None):
        container_config['HostConfig'] = {}

    nvidia_config = get_nvidia_configuration()

    # Setup the Volumes
    container_config['HostConfig'].setdefault('VolumeDriver', nvidia_config['VolumeDriver'])
    container_config['HostConfig'].setdefault('Binds', [])
    container_config['HostConfig']['Binds'].extend(nvidia_config['Volumes'])

    # Get nvidia control devices
    devices = container_config['HostConfig'].get('Devices', [])
    # suport both '0 1' and '0, 1' formats, just like nvidia-docker
    gpu_isolation = os.getenv('NV_GPU', '').replace(',', ' ').split()
    pattern = re.compile(r'/nvidia([0-9]+)$')
    for device in nvidia_config['Devices']:
        if gpu_isolation:
{'fraction_non_alphanumeric': 0.12313104661389622, 'fraction_numerical': 0.007036059806508356, 'mean_word_length': 4.41547619047619, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '657011', 'n_tokens_mistral': 1478, 'n_tokens_neox': 1353, 'n_words': 360}