<commit_before>"""This is an example of json schema for the GDC using schemas defined
in local yaml files (see gdcdictionary/examples).

This test class is making sure the example jsons comply with the schema.

Note this is NOT testing that the schema is sane. Just that we adhere
to it

"""

from utils import DATA_DIR
from jsonschema import validate, ValidationError

import json
import glob
import os
import pytest

def get_all_docs(subdir):
    for path in glob.glob(os.path.join(DATA_DIR, subdir, '*.json')):
        with open(path, 'r') as f:
            doc = json.load(f)
            if isinstance(doc, dict):
                yield doc
            elif isinstance(doc, list):
                for entity in doc:
                    yield entity
            else:
                assert False, "Error collecting file: {}".format(f)


@pytest.mark.parametrize("doc", get_all_docs("valid"))
def test_valid_examples(doc, schema):
    validate(doc, schema[doc["type"]])

@pytest.mark.parametrize("doc", get_all_docs("invalid"))
def test_invalid_examples(doc, schema):
    with pytest.raises(ValidationError):
        validate(doc, schema[doc["type"]])<commit_msg>feat(id): create ids for paremterization<commit_after>"""This is an example of json schema for the GDC using schemas defined
in local yaml files (see gdcdictionary/examples).

This test class is making sure the example jsons comply with the schema.

Note this is NOT testing that the schema is sane. Just that we adhere
to it

"""

from utils import DATA_DIR
from jsonschema import validate, ValidationError

import json
import glob
import os
import pytest


def get_all_docs(subdir):
    for path in sorted(glob.glob(os.path.join(DATA_DIR, subdir, '*.json'))):
        with open(path, 'r') as f:
            doc = json.load(f)
            if isinstance(doc, dict):
                yield doc
            elif isinstance(doc, list):
                if len(doc) != 1:
                    assert False, "Mismatch between files and documents in {}".format(f.name)
                for entity in doc:
                    yield entity
            else:
                assert False, "Error collecting file: {}".format(f)

def get_all_files(subdir):
    file_names = []
    for path in sorted(glob.glob(os.path.join(DATA_DIR, subdir, '*.json'))):
        with open(path, 'r') as f:
            file_names.append(f.name)
    return file_names


@pytest.mark.parametrize("doc", get_all_docs("valid"), ids=get_all_files("valid"))
def test_valid_examples(doc, schema):
    validate(doc, schema[doc["type"]])

@pytest.mark.parametrize("doc", get_all_docs("invalid"), ids=get_all_files("invalid"))
def test_invalid_examples(doc, schema):
    with pytest.raises(ValidationError):
        validate(doc, schema[doc["type"]])
{'fraction_non_alphanumeric': 0.10642135642135642, 'fraction_numerical': 0.00036075036075036075, 'mean_word_length': 3.0246734397677795, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '900964', 'n_tokens_mistral': 838, 'n_tokens_neox': 821, 'n_words': 268}