<commit_before>

from builtins import str
import os
import re
import math
from string import Template
# from datetime import timedelta
from .mycluster import check_output
from .mycluster import get_timedelta
from .mycluster import get_data
from .mycluster import load_template

from jinja2 import Environment, FileSystemLoader


"""
sacctmgr show cluster
"""


def scheduler_type():
    return 'slurm'


def name():
    with os.popen('sacctmgr show cluster') as f:
        f.readline()
        f.readline()
        return f.readline().strip().split(' ')[0]


def accounts():
    account_list = []

    with os.popen('sacctmgr --noheader list assoc user=`id -un` format=Account') as f:
        for line in f:
            account_list.append(line)

    return account_list


def queues():
    queue_list = []

    with os.popen('sinfo -sh') as f:
        for line in f:
            q = line.split(' ')[0].strip().replace("*", "")
            queue_list.append(q)

    return queue_list


def available_tasks(queue_id):

    # split queue id into queue and parallel env
    # list free slots
    free_tasks = 0
    max_tasks = 0
    queue_name = queue_id
    nc = node_config(queue_id)
    with os.popen('sinfo -sh -p ' + queue_name) as f:
        line = f.readline()
        new_line = re.sub(' +', ' ', line.strip())
        line = new_line.split(' ')[3]
        free_tasks = int(line.split('/')[1]) * nc['max task']
        max_tasks = int(line.split('/')[3]) * nc['max task']

    return {'available': free_tasks, 'max tasks': max_tasks}


def tasks_per_node(queue_id):
    queue_name = queue_id
    tasks = 0
    with os.popen('sinfo -Nelh -p ' + queue_name) as f:
        line = f.readline()
        new_line = re.sub(' +', ' ', line.strip())
        tasks = int(new_line.split(' ')[4])
    return tasks


def node_config(queue_id):
    # Find first node with queue and record node config
    queue_name = queue_id
    tasks = 0
    config = {}
    with os.popen('sinfo -Nelh -p ' + queue_name) as f:
        line = f.readline()
        if len(line):
            new_line = re.sub(' +', ' ', line.strip())
            tasks = int(new_line.split(' ')[4])
            memory = int(new_line.split(' ')[6])
            config['max task'] = tasks
            config['max thread'] = tasks
            config['max memory'] = memory
        else:
            raise Exception(
                "Requested partition %s has no nodes" % queue_name)

    return config


def create_submit(queue_id, **kwargs):

    queue_name = queue_id

    num_tasks = 1
    if 'num_tasks' in kwargs:
        num_tasks = kwargs['num_tasks']
    else:
        raise ValueError("num_tasks must be specified")

    tpn = tasks_per_node(queue_id)
    queue_tpn = tpn
    if 'tasks_per_node' in kwargs:
        tpn = kwargs['tasks_per_node']

    if 'num_threads_per_task' in kwargs:
        num_threads_per_task = kwargs['num_threads_per_task']
    else:
        raise ValueError("num_threads_per_task must be specified")

    my_name = kwargs.get('my_name', "myclusterjob")
    my_output = kwargs.get('my_output', "myclusterjob.out")
    my_script = kwargs.get('my_script', None)

    if 'mycluster-' in my_script:
        my_script = get_data(my_script)

    user_email = kwargs.get('user_email', None)
    project_name = kwargs.get('project_name', 'default')

    wall_clock = kwargs.get('wall_clock', '12:00:00')
    if ':' not in wall_clock:
        wall_clock = wall_clock + ':00:00'

    num_nodes = int(math.ceil(float(num_tasks) / float(tpn)))

    num_queue_slots = num_nodes * queue_tpn

    no_syscribe = kwargs.get('no_syscribe', False)

    record_job = not no_syscribe

    openmpi_args = kwargs.get('openmpi_args', "-bysocket -bind-to-socket")

    qos = kwargs.get('qos', None)

    template = load_template('slurm.jinja')

    script_str = template.render(my_name=my_name,
                                 my_script=my_script,
                                 my_output=my_output,
                                 user_email=user_email,
                                 queue_name=queue_name,
                                 num_queue_slots=num_queue_slots,
                                 num_tasks=num_tasks,
                                 tpn=tpn,
                                 num_threads_per_task=num_threads_per_task,
                                 num_nodes=num_nodes,
                                 project_name=project_name,
                                 wall_clock=wall_clock,
                                 record_job=record_job,
                                 openmpi_args=openmpi_args,
                                 qos=qos)
    return script_str


def submit(script_name, immediate, depends_on=None,<commit_msg>if tasks_per_node supplied as kwarg use that and don't check sinfo output<commit_after>

from builtins import str
import os
import re
import math
from string import Template
# from datetime import timedelta
from .mycluster import check_output
from .mycluster import get_timedelta
from .mycluster import get_data
from .mycluster import load_template

from jinja2 import Environment, FileSystemLoader


"""
sacctmgr show cluster
"""


def scheduler_type():
    return 'slurm'


def name():
    with os.popen('sacctmgr show cluster') as f:
        f.readline()
        f.readline()
        return f.readline().strip().split(' ')[0]


def accounts():
    account_list = []

    with os.popen('sacctmgr --noheader list assoc user=`id -un` format=Account') as f:
        for line in f:
            account_list.append(line)

    return account_list


def queues():
    queue_list = []

    with os.popen('sinfo -sh') as f:
        for line in f:
            q = line.split(' ')[0].strip().replace("*", "")
            queue_list.append(q)

    return queue_list


def available_tasks(queue_id):

    # split queue id into queue and parallel env
    # list free slots
    free_tasks = 0
    max_tasks = 0
    queue_name = queue_id
    nc = node_config(queue_id)
    with os.popen('sinfo -sh -p ' + queue_name) as f:
        line = f.readline()
        new_line = re.sub(' +', ' ', line.strip())
        line = new_line.split(' ')[3]
        free_tasks = int(line.split('/')[1]) * nc['max task']
        max_tasks = int(line.split('/')[3]) * nc['max task']

    return {'available': free_tasks, 'max tasks': max_tasks}


def tasks_per_node(queue_id):
    queue_name = queue_id
    tasks = 0
    with os.popen('sinfo -Nelh -p ' + queue_name) as f:
        line = f.readline()
        new_line = re.sub(' +', ' ', line.strip())
        tasks = int(new_line.split(' ')[4])
    return tasks


def node_config(queue_id):
    # Find first node with queue and record node config
    queue_name = queue_id
    tasks = 0
    config = {}
    with os.popen('sinfo -Nelh -p ' + queue_name) as f:
        line = f.readline()
        if len(line):
            new_line = re.sub(' +', ' ', line.strip())
            tasks = int(new_line.split(' ')[4])
            memory = int(new_line.split(' ')[6])
            config['max task'] = tasks
            config['max thread'] = tasks
            config['max memory'] = memory
        else:
            raise Exception(
                "Requested partition %s has no nodes" % queue_name)

    return config


def create_submit(queue_id, **kwargs):

    queue_name = queue_id

    num_tasks = 1
    if 'num_tasks' in kwargs:
        num_tasks = kwargs['num_tasks']
    else:
        raise ValueError("num_tasks must be specified")

    if 'tasks_per_node' in kwargs:
        tpn = kwargs['tasks_per_node']
    else:
        tpn = tasks_per_node(queue_id)
    if 'num_threads_per_task' in kwargs:
        num_threads_per_task = kwargs['num_threads_per_task']
    else:
        raise ValueError("num_threads_per_task must be specified")

    my_name = kwargs.get('my_name', "myclusterjob")
    my_output = kwargs.get('my_output', "myclusterjob.out")
    my_script = kwargs.get('my_script', None)

    if 'mycluster-' in my_script:
        my_script = get_data(my_script)

    user_email = kwargs.get('user_email', None)
    project_name = kwargs.get('project_name', 'default')

    wall_clock = kwargs.get('wall_clock', '12:00:00')
    if ':' not in wall_clock:
        wall_clock = wall_clock + ':00:00'

    num_nodes = int(math.ceil(float(num_tasks) / float(tpn)))

    no_syscribe = kwargs.get('no_syscribe', False)

    record_job = not no_syscribe

    openmpi_args = kwargs.get('openmpi_args', "-bysocket -bind-to-socket")

    qos = kwargs.get('qos', None)

    template = load_template('slurm.jinja')

    script_str = template.render(my_name=my_name,
                                 my_script=my_script,
                                 my_output=my_output,
                                 user_email=user_email,
                                 queue_name=queue_name,
                                 num_tasks=num_tasks,
                                 tpn=tpn,
                                 num_threads_per_task=num_threads_per_task,
                                 num_nodes=num_nodes,
                                 project_name=project_name,
                                 wall_clock=wall_clock,
                                 record_job=record_job,
                                 openmpi_args=openmpi_args,
                                 qos=qos)
    return script_str


def submit(script_name, immediate, depends_on=None,
{'fraction_non_alphanumeric': 0.1099893730074389, 'fraction_numerical': 0.005100956429330499, 'mean_word_length': 2.117257369990063, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4467986', 'n_tokens_mistral': 2991, 'n_tokens_neox': 2917, 'n_words': 691}