<commit_before>import sys
from pprint import pprint

import yaml

sys.path.insert(0, '../../src')
sys.path.insert(0, '../..')

from tests.functional.test_yaml import convert_to_dict


def check_dict(d, required=None, optional=None):
    assert isinstance(d, dict)
    if required is None:
        required = []
    if optional is None:
        optional = []
    for key in required:
        if key not in d:
            sys.exit("key %r is required" % key)
    allowed_keys = set(required)
    allowed_keys.update(optional)
    for key in d.keys():
        if key not in allowed_keys:
            sys.exit("key %r is not allowed.  Allowed keys are: %r" %
                     (key, allowed_keys))


def lint_case(case, verbose=False):
    if verbose:
        print("--- linting case ---")
        pprint(case)

    check_dict(case, optional=['available', 'request', 'response', 'skip'])
    available = case.get("available", [])
    requests = case.get("request", [])
    responses = case.get("response", [])
    assert isinstance(available, list)
    assert isinstance(requests, list)
    assert isinstance(responses, list)
    assert len(requests) == len(responses)

    for package in available:
        if isinstance(package, str):
            package = convert_to_dict(package)
        assert isinstance(package, dict)

    for request in requests:
        check_dict(request, optional=['install', 'uninstall', 'options'])

    for response in responses:
        check_dict(response, optional=['state', 'conflicting'])
        assert len(response) == 1


def lint_yml(yml_file, verbose=False):
    if verbose:
        print("=== linting: %s ===" % yml_file)
    assert yml_file.endswith(".yml")
    with open(yml_file) as fi:
        data = yaml.safe_load(fi)
    if verbose:
        pprint(data)

    check_dict(data, required=['cases'], optional=['base'])
    base = data.get("base", {})
    cases = data["cases"]
    for i, case_template in enumerate(cases):
        case = base.copy()
        case.update(case_template)
        lint_case(case, verbose)
<commit_msg>simplify code, ensure state is a list<commit_after>import sys
from pprint import pprint

import yaml

sys.path.insert(0, '../../src')
sys.path.insert(0, '../..')

from tests.functional.test_yaml import convert_to_dict


def check_dict(d, required=None, optional=None):
    assert isinstance(d, dict)
    if required is None:
        required = []
    if optional is None:
        optional = []
    for key in required:
        if key not in d:
            sys.exit("key %r is required" % key)
    allowed_keys = set(required)
    allowed_keys.update(optional)
    for key in d.keys():
        if key not in allowed_keys:
            sys.exit("key %r is not allowed.  Allowed keys are: %r" %
                     (key, allowed_keys))


def lint_case(case, verbose=False):
    if verbose:
        print("--- linting case ---")
        pprint(case)

    check_dict(case, optional=['available', 'request', 'response', 'skip'])
    available = case.get("available", [])
    requests = case.get("request", [])
    responses = case.get("response", [])
    assert isinstance(available, list)
    assert isinstance(requests, list)
    assert isinstance(responses, list)
    assert len(requests) == len(responses)

    for package in available:
        if isinstance(package, str):
            package = convert_to_dict(package)
        assert isinstance(package, dict)

    for request, response in zip(requests, responses):
        check_dict(request, optional=['install', 'uninstall', 'options'])
        check_dict(response, optional=['state', 'conflicting'])
        assert len(response) == 1
        assert isinstance(response.get('state') or [], list)


def lint_yml(yml_file, verbose=False):
    if verbose:
        print("=== linting: %s ===" % yml_file)
    assert yml_file.endswith(".yml")
    with open(yml_file) as fi:
        data = yaml.safe_load(fi)
    if verbose:
        pprint(data)

    check_dict(data, required=['cases'], optional=['base'])
    base = data.get("base", {})
    cases = data["cases"]
    for i, case_template in enumerate(cases):
        case = base.copy()
        case.update(case_template)
        lint_case(case, verbose)

{'fraction_non_alphanumeric': 0.13437426070499173, 'fraction_numerical': 0.0014194464158978, 'mean_word_length': 2.8227848101265822, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6160736', 'n_tokens_mistral': 1305, 'n_tokens_neox': 1262, 'n_words': 344}