<commit_before>from unittest import skipUnless, TestCase, skip

from scripts.service.ws_re.register.registers import Registers
from tools import INTEGRATION_TEST


_MAX_SIZE_WIKI_PAGE = 2_098_175


@skipUnless(INTEGRATION_TEST, "only execute in integration test")
class TestIntegrationRegister(TestCase):
    @classmethod
    def setUpClass(cls):
        cls.registers = Registers()

    def test_length_of_alphabetic(self):
        for register in self.registers.alphabetic.values():
            self.assertGreater(_MAX_SIZE_WIKI_PAGE, len(register.get_register_str()), f"register {register} is now to big.")

    def test_previous_next_in_order(self):
        errors = []
        for register in self.registers.volumes.values():
            for i, lemma in enumerate(register):
                pre_lemma = register[i -1] if i > 0 else None
                if pre_lemma and pre_lemma["next"]:
                    if not pre_lemma["next"] == lemma["lemma"]:  # pragma: no cover
                        errors.append(f"PRE lemma name {lemma['lemma']} /{i} in register {register.volume.name} not the same as pre lemma")
                try:
                    post_lemma = register[i + 1]
                    if post_lemma and post_lemma["previous"]:
                        if not post_lemma["previous"] == lemma["lemma"]:  # pragma: no cover
                            errors.append(f"POST lemma name {lemma['lemma']} /{i} in register {register.volume.name} not the same as post lemma")
                except IndexError:
                    pass

        if errors:  # pragma: no cover
            count = len(errors)
            errors.insert(0, f"COUNT ERRORS: {count}")<commit_msg>test if all mappings has a target<commit_after>from unittest import skipUnless, TestCase, skip

from scripts.service.ws_re.register.author import Authors
from scripts.service.ws_re.register.registers import Registers
from tools import INTEGRATION_TEST


_MAX_SIZE_WIKI_PAGE = 2_098_175

class TestAuthors(TestCase):
    @classmethod
    def setUpClass(cls) -> None:
        cls.authors = Authors()

    def test_all_mappings_have_target(self):
        errors = []
        authors_in_mappings = set()
        for mapping in self.authors._mapping.values():
            if isinstance(mapping, str):
                authors_in_mappings.add(mapping)
            else:
                for author in mapping.values():
                    authors_in_mappings.add(author)
        for author in authors_in_mappings:
            try:
                self.authors.get_author(author)
            except KeyError:
                errors.append(f"Mapping target {author} not there.")
        if errors:  # pragma: no cover
            count = len(errors)
            errors.insert(0, f"COUNT ERRORS: {count}")
            errors.append(f"COUNT ERRORS: {count}")
            raise AssertionError("\n".join(errors))



@skipUnless(INTEGRATION_TEST, "only execute in integration test")
class TestIntegrationRegister(TestCase):
    @classmethod
    def setUpClass(cls):
        cls.registers = Registers()

    def test_length_of_alphabetic(self):
        for register in self.registers.alphabetic.values():
            self.assertGreater(_MAX_SIZE_WIKI_PAGE, len(register.get_register_str()), f"register {register} is now to big.")

    def test_previous_next_in_order(self):
        errors = []
        for register in self.registers.volumes.values():
            for i, lemma in enumerate(register):
                pre_lemma = register[i -1] if i > 0 else None
                if pre_lemma and pre_lemma["next"]:
                    if not pre_lemma["next"] == lemma["lemma"]:  # pragma: no cover
                        errors.append(f"PRE lemma name {lemma['lemma']} /{i} in register {register.volume.name} not the same as pre lemma")
                try:
                    post_lemma = register[i + 1]
                    if post_lemma and post_lemma["previous"]:
                        if not post_lemma["previous"] == lemma["lemma"]:  # pragma: no cover
                            errors.append(f"POST lemma name {lemma['lemma']} /{i} in register {register.volume.name} not the same as post lemma")
                except IndexError:
                    pass

        if errors:  # pragma: no cover
            count = len(errors)
            errors.insert(0, f"COUNT ERRORS: {count}")
{'fraction_non_alphanumeric': 0.09688502061383417, 'fraction_numerical': 0.005267979844251031, 'mean_word_length': 2.2492559523809526, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2974425', 'n_tokens_mistral': 1263, 'n_tokens_neox': 1192, 'n_words': 327}