<commit_before>def gpu_queue(options):
    """
    Queued up containers waiting for GPU resources
    """
    import docker
    import json
    import time

    from vent.helpers.meta import GpuUsage

    status = (False, None)

    print("gpu queue", str(options))
    print("gpu queue", str(GpuUsage(base_dir="/vent/",
                                    meta_dir="/vent")))

    options = json.loads(options)
    configs = options['configs']
    gpu_options = configs['gpu_options']
    devices = []

    print(str(configs['devices']))
    # device specified, remove all other devices
    if 'device' in gpu_options:
        dev = '/dev/nvidia' + gpu_options['device'] + ':/dev/nvidia'
        dev += gpu_options['device'] + ':rwm'
        if 'devices' in configs:
            d = list(configs['devices'])
            print(str(d))
            for device in d:
                print(dev + " compared to " + device)
                if any(str.isdigit(str(char)) for char in device):
                    if dev == device:
                        devices.append(device)
                    else:
                        print(dev + " doesn't match, removing: " + device)
                        configs['devices'].remove(device)
    else:
        d = configs['devices']
        for device in d:
            if any(str.isdigit(str(char)) for char in device):
                devices.append(device)

    # check if devices is still an empty list
    if not devices:
        status = (False, "no valid devices match the requested device")
        print(str(status))
        return status

    mem_needed = 0
    dedicated = False
    # need a gpu to itself
    if ('dedicated' in configs['gpu_options'] and
       configs['gpu_options']['dedicated'] == 'yes'):
        dedicated = True
    if 'mem_mb' in configs['gpu_options']:
        # TODO input error checking
        mem_needed = int(configs['gpu_options']['mem_mb'])

    print("mem_needed: ", mem_needed)
    print("dedicated: ", dedicated)
    device = None
    while not device:
        usage = GpuUsage(base_dir="/vent/", meta_dir="/vent")

        if usage[0]:
            usage = usage[1]
        else:
            return usage
        print(usage)
        # {"device": "0",
        #  "mem_mb": "1024",
        #  "dedicated": "yes",
        #  "enabled": "yes"}
        for d in devices:
            dev = str(d.split(":")[0].split('nvidia')[1])
            print(dev)
            # if the device is already dedicated, can't be used
            dedicated_gpus = usage['vent_usage']['dedicated']
            is_dedicated = False
            for gpu in dedicated_gpus:
                if dev in gpu:
                    is_dedicated = True
            print("is_dedicated: ", is_dedicated)
            if not is_dedicated:
                ram_used = 0
                if dev in usage['vent_usage']['mem_mb']:
                    ram_used = usage['vent_usage']['mem_mb'][dev]
                # check for vent usage/processes running
                if (dedicated and
                   dev not in usage['vent_usage']['mem_mb'] and
                   mem_needed <= usage[int(dev)]['global_memory'] and
                   not usage[int(dev)]['processes']):
                    device = dev
                # check for ram constraints
                elif mem_needed <= (usage[int(dev)]['global_memory'] - ram_used):
                    device = dev

        # TODO make this sleep incremental up to a point, potentially kill
        #      after a set time configured from vent.cfg, outputting as it goes
        time.sleep(1)

    # lock jobs to a specific gpu (no shared GPUs for a single process) this is
    # needed to calculate if memory requested (but not necessarily in use)
    # would become oversubscribed

    # store which device was mapped
    options['labels']['vent.gpu.device'] = device
    gpu_device = '/dev/nvidia' + device + ':/dev/nvidia' + device + ':rwm'
    if 'devices' in configs:
        d = configs['devices']
        for dev in d:
            if any(str.isdigit(str(char)) for char in dev):
                if gpu_device != dev:
                    configs['devices'].remove(dev)

    try:
        d_client = docker.from_env()
        del options['configs']
        del configs['gpu_options']
        params = options.copy()
        params.update(configs)
        print(str(params))
        d_client.containers.run(**params)
        status = (True, None)
    except Exception as e:  # pragma: no cover
        status = (False, str(e))

    print(str(status))
    return status


def file_queue(path, template_path="/vent/", r_host="redis"):
    """
    Processes files that have been added from the rq-worker, starts plugins
    that match the mime type for the new file.
    """
    import ConfigParser
    import ast
    import docker
    import json
    import requests
    import os
    import sys

    from redis import Redis
    from rq import Queue
    from subprocess import check_output, Popen, PIPE
    from string import punctuation
    from vent.helpers.logs import Logger

    status = (True, None)
    images = []
    configs = {}
    logger = Logger()
    try:
        d_client = docker.from_env()

        # get the correct path for binding
        vent_config = ConfigParser.RawConfigParser()
        vent_config.optionxform = str
        vent_config.read(template_path+'vent.cfg')
        if (vent_config.has_section('main') and
           vent_config.has_option('main', 'files')):
            files = vent_config.get('main', 'files')
        else:
            files = '/'

        # deal with ~<commit_msg> missed name of one logger<commit_after>def gpu_queue(options):
    """
    Queued up containers waiting for GPU resources
    """
    import docker
    import json
    import time

    from vent.helpers.meta import GpuUsage

    status = (False, None)

    print("gpu queue", str(options))
    print("gpu queue", str(GpuUsage(base_dir="/vent/",
                                    meta_dir="/vent")))

    options = json.loads(options)
    configs = options['configs']
    gpu_options = configs['gpu_options']
    devices = []

    print(str(configs['devices']))
    # device specified, remove all other devices
    if 'device' in gpu_options:
        dev = '/dev/nvidia' + gpu_options['device'] + ':/dev/nvidia'
        dev += gpu_options['device'] + ':rwm'
        if 'devices' in configs:
            d = list(configs['devices'])
            print(str(d))
            for device in d:
                print(dev + " compared to " + device)
                if any(str.isdigit(str(char)) for char in device):
                    if dev == device:
                        devices.append(device)
                    else:
                        print(dev + " doesn't match, removing: " + device)
                        configs['devices'].remove(device)
    else:
        d = configs['devices']
        for device in d:
            if any(str.isdigit(str(char)) for char in device):
                devices.append(device)

    # check if devices is still an empty list
    if not devices:
        status = (False, "no valid devices match the requested device")
        print(str(status))
        return status

    mem_needed = 0
    dedicated = False
    # need a gpu to itself
    if ('dedicated' in configs['gpu_options'] and
       configs['gpu_options']['dedicated'] == 'yes'):
        dedicated = True
    if 'mem_mb' in configs['gpu_options']:
        # TODO input error checking
        mem_needed = int(configs['gpu_options']['mem_mb'])

    print("mem_needed: ", mem_needed)
    print("dedicated: ", dedicated)
    device = None
    while not device:
        usage = GpuUsage(base_dir="/vent/", meta_dir="/vent")

        if usage[0]:
            usage = usage[1]
        else:
            return usage
        print(usage)
        # {"device": "0",
        #  "mem_mb": "1024",
        #  "dedicated": "yes",
        #  "enabled": "yes"}
        for d in devices:
            dev = str(d.split(":")[0].split('nvidia')[1])
            print(dev)
            # if the device is already dedicated, can't be used
            dedicated_gpus = usage['vent_usage']['dedicated']
            is_dedicated = False
            for gpu in dedicated_gpus:
                if dev in gpu:
                    is_dedicated = True
            print("is_dedicated: ", is_dedicated)
            if not is_dedicated:
                ram_used = 0
                if dev in usage['vent_usage']['mem_mb']:
                    ram_used = usage['vent_usage']['mem_mb'][dev]
                # check for vent usage/processes running
                if (dedicated and
                   dev not in usage['vent_usage']['mem_mb'] and
                   mem_needed <= usage[int(dev)]['global_memory'] and
                   not usage[int(dev)]['processes']):
                    device = dev
                # check for ram constraints
                elif mem_needed <= (usage[int(dev)]['global_memory'] - ram_used):
                    device = dev

        # TODO make this sleep incremental up to a point, potentially kill
        #      after a set time configured from vent.cfg, outputting as it goes
        time.sleep(1)

    # lock jobs to a specific gpu (no shared GPUs for a single process) this is
    # needed to calculate if memory requested (but not necessarily in use)
    # would become oversubscribed

    # store which device was mapped
    options['labels']['vent.gpu.device'] = device
    gpu_device = '/dev/nvidia' + device + ':/dev/nvidia' + device + ':rwm'
    if 'devices' in configs:
        d = configs['devices']
        for dev in d:
            if any(str.isdigit(str(char)) for char in dev):
                if gpu_device != dev:
                    configs['devices'].remove(dev)

    try:
        d_client = docker.from_env()
        del options['configs']
        del configs['gpu_options']
        params = options.copy()
        params.update(configs)
        print(str(params))
        d_client.containers.run(**params)
        status = (True, None)
    except Exception as e:  # pragma: no cover
        status = (False, str(e))

    print(str(status))
    return status


def file_queue(path, template_path="/vent/", r_host="redis"):
    """
    Processes files that have been added from the rq-worker, starts plugins
    that match the mime type for the new file.
    """
    import ConfigParser
    import ast
    import docker
    import json
    import requests
    import os
    import sys

    from redis import Redis
    from rq import Queue
    from subprocess import check_output, Popen, PIPE
    from string import punctuation
    from vent.helpers.logs import Logger

    status = (True, None)
    images = []
    configs = {}
    logger = Logger(__name__)
    try:
        d_client = docker.from_env()

        # get the correct path for binding
        vent_config = ConfigParser.RawConfigParser()
        vent_config.optionxform = str
        vent_config.read(template_path+'vent.cfg')
        if (vent_config.has_section('main') and
           vent_config.has_option('main', 'files')):
            files = vent_config.get('main', 'files')
        else:
            files = '/'

        # deal with ~
{'fraction_non_alphanumeric': 0.10884837705646955, 'fraction_numerical': 0.00213428190306803, 'mean_word_length': 1.9316996871741396, 'pattern_counts': {'":': 10, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2734850', 'n_tokens_mistral': 3239, 'n_tokens_neox': 3108, 'n_words': 921}