<commit_before>import networkx as nx
import os
import tempfile

from ds_commons import util
from scheduler.exceptions import _log_raise, _log_raise_if, DAGMisconfigured

from .constants import DEPENDENCY_GROUP_DEFAULT_NAME, JOB_ID_DEFAULT_TEMPLATE
from . import node


def _validate_dep_grp_metadata(dep_info, ld, tasks_dct, dep_name):
    ld1 = ld
    _template, child_template = node.get_job_id_template(ld['app_name'])
    for parent_app_name in dep_info['app_name']:
        ld = dict(
            parent_app_name=parent_app_name,
            dependency_group="depends_on.%s" % dep_name, **ld1)

        _log_raise_if(
            parent_app_name not in tasks_dct,
            "Unrecognized parent_app_name in a `depends_on` dependency group",
            extra=ld,
            exception_kls=DAGMisconfigured)
        if len(dep_info) == 1:
            _, parent_template = node.get_job_id_template(parent_app_name)
            _log_raise_if(
                not set(child_template).issuperset(parent_template),
                ("If you choose specify a dependency with no metadata, then"
                 " the"
                 " child task's job_id must be a superset of the metadata in"
                 " the parent's job_id. Otherwise, (I think) there"
                 " are cases where you cannot identify"
                 " a parent job_id given a child job_id."),
                extra=dict(
                    parent_job_id_template=parent_template,
                    child_job_id_template=child_template,
                    **ld),
                exception_kls=DAGMisconfigured)
    for k, v in dep_info.items():
        _log_raise_if(
            len(set(v)) != len(v),
            "You have duplicate metadata in dependency group metadata",
            extra=dict(key=k, value=v, **ld),
            exception_kls=DAGMisconfigured)

    for k in child_template:
        if len(dep_info) == 1 or 'job_id' in dep_info:
            break
        if k != 'dependency_group_name':
            _log_raise_if(
                k not in dep_info,
                ("The dependency group must specify all fields specified in"
                    " the job_id template"),
                extra=dict(key=k, job_id_template=_template, **ld),
                exception_kls=DAGMisconfigured)
            _log_raise_if(
                len(dep_info[k]) != 1,
                ("You cannot specify more than 1 value for each field"
                    " specified in your task's job_id template."),
                extra=dict(key=k, value=v, job_id_template=_template, **ld),
                exception_kls=DAGMisconfigured)


def _validate_dependency_groups_part2(dep_name, dep_info, ld, tasks_dct):
    _log_raise_if(
        ("app_name" not in dep_info
         or not isinstance(dep_info["app_name"], list)),
        ("Each dependency group the task depends on must specify"
         " an app_name key whose value is a list"),
        extra=dict(
            key="depends_on", invalid_dependency_group=dep_name,
            dep_info=str(dep_info), **ld),
        exception_kls=DAGMisconfigured)
    _validate_dep_grp_metadata(
        dep_info, ld=ld, tasks_dct=tasks_dct, dep_name=dep_name)


def _validate_dependency_groups(tasks_dct, metadata, ld):
    for dep_name, dep_info in metadata.get("depends_on", {}).items():
        _log_raise_if(
            dep_name in tasks_dct,
            ("Task's depends_on value has a naming conflict. You cannot"
             " identify a dependency group with the same name as an"
             " app_name."),
            extra=dict(
                key="depends_on",
                invalid_dependency_group=dep_name,
                **ld),
            exception_kls=DAGMisconfigured)
        if isinstance(dep_info, list):
            for _dep_info in dep_info:
                _validate_dependency_groups_part2(
                    dep_name, _dep_info, ld, tasks_dct)
        else:
            _validate_dependency_groups_part2(
                dep_name, dep_info, ld, tasks_dct)

<commit_msg>adds another check for invalid tasks.json configuration<commit_after>import networkx as nx
import os
import tempfile

from ds_commons import util
from scheduler.exceptions import _log_raise, _log_raise_if, DAGMisconfigured

from .constants import DEPENDENCY_GROUP_DEFAULT_NAME, JOB_ID_DEFAULT_TEMPLATE
from . import node


def _validate_dep_grp_metadata(dep_info, ld, tasks_dct, dep_name):
    ld1 = ld
    _template, child_template = node.get_job_id_template(ld['app_name'])
    for parent_app_name in dep_info['app_name']:
        ld = dict(
            parent_app_name=parent_app_name,
            dependency_group="depends_on.%s" % dep_name, **ld1)

        _log_raise_if(
            parent_app_name not in tasks_dct,
            "Unrecognized parent_app_name in a `depends_on` dependency group",
            extra=ld,
            exception_kls=DAGMisconfigured)
        if len(dep_info) == 1:
            _, parent_template = node.get_job_id_template(parent_app_name)
            _log_raise_if(
                not set(child_template).issuperset(parent_template),
                ("If you choose specify a dependency with no metadata, then"
                 " the"
                 " child task's job_id must be a superset of the metadata in"
                 " the parent's job_id. Otherwise, there"
                 " are cases where you cannot identify"
                 " a parent job_id given a child job_id."),
                extra=dict(
                    parent_job_id_template=parent_template,
                    child_job_id_template=child_template,
                    **ld),
                exception_kls=DAGMisconfigured)
    for k, v in dep_info.items():
        _log_raise_if(
            len(set(v)) != len(v),
            "You have duplicate metadata in dependency group metadata",
            extra=dict(key=k, value=v, **ld),
            exception_kls=DAGMisconfigured)

    for k in child_template:
        if len(dep_info) == 1 or 'job_id' in dep_info:
            break
        if k != 'dependency_group_name':
            _log_raise_if(
                k not in dep_info,
                ("The dependency group must specify all fields specified in"
                    " the job_id template"),
                extra=dict(key=k, job_id_template=_template, **ld),
                exception_kls=DAGMisconfigured)
            _log_raise_if(
                len(dep_info[k]) != 1,
                ("You cannot specify more than 1 value for each field"
                    " specified in your task's job_id template."),
                extra=dict(key=k, value=v, job_id_template=_template, **ld),
                exception_kls=DAGMisconfigured)


def _validate_dependency_groups_part2(dep_name, dep_info, ld, tasks_dct):
    _log_raise_if(
        ("app_name" not in dep_info
         or not isinstance(dep_info["app_name"], list)),
        ("Each dependency group the task depends on must specify"
         " an app_name key whose value is a list"),
        extra=dict(
            key="depends_on", invalid_dependency_group=dep_name,
            dep_info=str(dep_info), **ld),
        exception_kls=DAGMisconfigured)
    _validate_dep_grp_metadata(
        dep_info, ld=ld, tasks_dct=tasks_dct, dep_name=dep_name)


def _validate_dependency_groups(tasks_dct, metadata, ld):
    for dep_name, dep_info in metadata.get("depends_on", {}).items():
        _log_raise_if(
            dep_name in tasks_dct,
            ("Task's depends_on value has a naming conflict. You cannot"
             " identify a dependency group with the same name as an"
             " app_name."),
            extra=dict(
                key="depends_on",
                invalid_dependency_group=dep_name,
                **ld),
            exception_kls=DAGMisconfigured)
        if isinstance(dep_info, list):
            for _dep_info in dep_info:
                _validate_dependency_groups_part2(
                    dep_name, _dep_info, ld, tasks_dct)
            # check job_id template metadata is consistent
            # across the dependency group
            for identifier in node.get_job_id_template(ld['app_name'])[1]:
                values = [_dep_info.get(identifier) for _dep_info in dep_info
                          if 'job_id' not in _dep_info]
                _log_raise_if(
                    not reduce(lambda x, y: x == y, values),
                    ("You specified inconsistent values for job_id"
                     " metadata.  Each sub-dependency in your dependency"
                     " group must specify the exact same metadata value"
                     " for each identifier in your app's job_id template"),
                    extra=dict(
                        key="depends_on", invalid_dependency_group=dep_name,
                        invalid_identifier=identifier,
                        values=values, **ld),
                    exception_kls=DAGMisconfigured)
        else:
            _validate_dependency_groups_part2(
                dep_name, dep_info, ld, tasks_dct)


{'fraction_non_alphanumeric': 0.0762870686804101, 'fraction_numerical': 0.0020945871458494104, 'mean_word_length': 2.0638297872340425, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4724708', 'n_tokens_mistral': 2695, 'n_tokens_neox': 2582, 'n_words': 666}