<commit_before>__author__ = 'sfranky'
import yaml
from collections import OrderedDict


def calculate_oar_state(jobid_state_lot, nr_of_jobs, node_state_mapping):
    """
    If all resource ids within the node are either alive or dead or suspected, the respective label is given to the node.
    Otherwise, a mixed-state is reported
    """
    states = [job_state_tpl[1] for job_state_tpl in jobid_state_lot]
    alive = states.count('Alive')
    dead = states.count('Dead')
    suspected = states.count('Suspected')

    if bool(alive) + bool(dead) + bool(suspected) > 1:
        state = node_state_mapping['mixed']
        return state
    else:
        return node_state_mapping[states[0]]


def read_oarnodes_yaml(fn_s, fn_y, write_method):
    nodes_resids = read_oarnodes_s_yaml(fn_s, write_method)
    # resids_jobs = read_oarnodes_y_yaml(fn_y, write_method)
    resids_jobs = read_oarnodes_y_textyaml(fn_y, write_method)

    nodes_jobs = {}
    for node in nodes_resids:
        resids_state_lot = nodes_resids[node]
        for (resid, state) in resids_state_lot:
            nodes_jobs.setdefault(node, []).append((resids_jobs[resid], state))

    worker_nodes = list()
    node_state_mapping = {'Alive': '-', 'Dead': 'd', 'Suspected': 's', 'Mixed': '%'}
    for node in nodes_jobs:
        d = OrderedDict()
        d['domainname'] = node
        nr_of_jobs = len(nodes_jobs[node])
        d['np'] = nr_of_jobs
        d['core_job_map'] = [{'core': idx, 'job': job[0]} for idx, job in enumerate(nodes_jobs[node]) if job[0] is not None]
        if not d['core_job_map']:
            del d['core_job_map']
        d['state'] = calculate_oar_state(nodes_jobs[node], nr_of_jobs, node_state_mapping)
        worker_nodes.append(d)

    return worker_nodes


def read_oarnodes_s_yaml(fn_s, write_method):
    with open(fn_s, mode='r') as fin:
        data = yaml.load(fin)
    nodes_resids = {node: resid_state.items() for node, resid_state in data.items()}
    return nodes_resids


def read_oarnodes_y_yaml(fn_y, write_method):
    with open('oarnodes_y', mode='r') as fin:
        data = yaml.load(fin)
    resids_jobs = {resid: info.get('jobs', None) for resid, info in data.items()}
    return resids_jobs


def read_oarnodes_y_textyaml(fn, write_method):
    oar_nodes = {}
    with open(fn, mode='r') as fin:<commit_msg>oarnodes_y read method controlled by external option, as for all the rest<commit_after>__author__ = 'sfranky'
import yaml
from collections import OrderedDict


def calculate_oar_state(jobid_state_lot, nr_of_jobs, node_state_mapping):
    """
    If all resource ids within the node are either alive or dead or suspected, the respective label is given to the node.
    Otherwise, a mixed-state is reported
    """
    states = [job_state_tpl[1] for job_state_tpl in jobid_state_lot]
    alive = states.count('Alive')
    dead = states.count('Dead')
    suspected = states.count('Suspected')

    if bool(alive) + bool(dead) + bool(suspected) > 1:
        state = node_state_mapping['mixed']
        return state
    else:
        return node_state_mapping[states[0]]


def read_oarnodes_y(fn_y, write_method):
    if write_method == 'yaml':
        return read_oarnodes_y_yaml(fn_y)
    else:
        return read_oarnodes_y_textyaml(fn_y)


def read_oarnodes_yaml(fn_s, fn_y, write_method):
    nodes_resids = read_oarnodes_s_yaml(fn_s, write_method)
    # resids_jobs = read_oarnodes_y_yaml(fn_y)
    # resids_jobs = read_oarnodes_y_textyaml(fn_y)
    resids_jobs = read_oarnodes_y(fn_y, write_method)

    nodes_jobs = {}
    for node in nodes_resids:
        resids_state_lot = nodes_resids[node]
        for (resid, state) in resids_state_lot:
            nodes_jobs.setdefault(node, []).append((resids_jobs[resid], state))

    worker_nodes = list()
    node_state_mapping = {'Alive': '-', 'Dead': 'd', 'Suspected': 's', 'Mixed': '%'}
    for node in nodes_jobs:
        d = OrderedDict()
        d['domainname'] = node
        nr_of_jobs = len(nodes_jobs[node])
        d['np'] = nr_of_jobs
        d['core_job_map'] = [{'core': idx, 'job': job[0]} for idx, job in enumerate(nodes_jobs[node]) if job[0] is not None]
        if not d['core_job_map']:
            del d['core_job_map']
        d['state'] = calculate_oar_state(nodes_jobs[node], nr_of_jobs, node_state_mapping)
        worker_nodes.append(d)

    return worker_nodes


def read_oarnodes_s_yaml(fn_s, write_method):
    with open(fn_s, mode='r') as fin:
        data = yaml.load(fin)
    nodes_resids = {node: resid_state.items() for node, resid_state in data.items()}
    return nodes_resids


def read_oarnodes_y_yaml(fn_y):
    with open('oarnodes_y', mode='r') as fin:
        data = yaml.load(fin)
    resids_jobs = {resid: info.get('jobs', None) for resid, info in data.items()}
    return resids_jobs


def read_oarnodes_y_textyaml(fn):
    oar_nodes = {}
    with open(fn, mode='r') as fin:
{'fraction_non_alphanumeric': 0.11591836734693878, 'fraction_numerical': 0.0020408163265306124, 'mean_word_length': 3.6192271442035815, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5634949', 'n_tokens_mistral': 1779, 'n_tokens_neox': 1719, 'n_words': 398}