<commit_before>import os
import json

from django.core.management.base import BaseCommand, CommandError
from web.MetaInfo import MetaInfo

class Command(BaseCommand):
    help = "Validates JSON language files with the meta language files"

    def handle(self, *args, **options):
        errors = []

        metainfo = MetaInfo()

        # Look though thesaurus directories, see if any files don't match
        # Load MetaInfo, check directories are in MetaInfo
        meta_files = os.listdir("web/thesauruses/_meta")

        for dir in os.listdir("web/thesauruses"):
            if dir == "_meta":
                continue
            path = "web/thesauruses/" + dir
            if os.path.isdir(path):

                if not dir in list(metainfo.languages.values()):
                    errors.append(path + " is not listed as a language in meta_info.json")

                files = os.listdir("web/thesauruses/" + dir)
                for f in files:
                    path = "web/thesauruses/" + dir + "/" + f
                    if not f in meta_files:
                        errors.append(path + " is not a valid concept filename")

        # Also check structures are in MetaInfo

        for meta_file in meta_files:
            structure_name = meta_file[:-5]
            if not structure_name in list(metainfo.data_structures.values()):
                errors.append("web/thesauruses/_meta/" + meta_file + " is not listed as a structure in meta_info.json")

        # Open up thesaurus directory
        language_dirs = os.listdir("web/thesauruses/")
        for lang_dir in language_dirs:
            if lang_dir == "_meta":
                continue
            if os.path.isfile("web/thesauruses/" + lang_dir):
                continue

        #    Open up each structures file
            structure_files = os.listdir("web/thesauruses/" + lang_dir)
            for structure_file in structure_files:
                structure = structure_file[:-5]
                metastructure = metainfo.structure(structure)

        #       Ensure valid lang/version/name
                meta_structure_file_path = os.path.join(
                    "web", "thesauruses", lang_dir, structure) + ".json"
                with open(meta_structure_file_path, 'r') as meta_structure_file:
                    data = meta_structure_file.read()
                    # parse file
                    meta_structure_file_json = json.loads(data)

                    language = meta_structure_file_json["meta"]["language"]
                    language_version = meta_structure_file_json["meta"]["language_version"]
                    language_name = meta_structure_file_json["meta"]["language_name"]
                    if not language:
                        errors.append(lang_dir + "/" + structure + ".json has an empty language attribute and needs to be updated")
                    elif language == "language_id":
                        errors.append(lang_dir + "/" + structure + ".json has the default language attribute and needs to be updated")
                    elif not language == lang_dir:
                        errors.append(lang_dir + "/" + structure + ".json has a language attribute that should be '" + lang_dir + "' and needs to be updated")

                    if not language_version:
                        errors.append(lang_dir + "/" + structure + ".json has an empty language_version attribute and needs to be updated")
                    elif language_version == "version.number":
                        errors.append(lang_dir + "/" + structure + ".json has the default language_version attribute and needs to be updated")

                    if not language_name:
                        errors.append(lang_dir + "/" + structure + ".json has an empty language_name attribute and needs to be updated")
                    elif language_name == "Human-Friendly Language Name":
                        errors.append(lang_dir + "/" + structure + ".json has the default language_name attribute and needs to be updated")

        #       Ensure categories aren't in file
        #       Ensure name lines are removed
        #       Ensure there's either code or not-implemented
        #       Ensure if not-implemented, there's no code line
        #       Ensure if code, there's no not-implemented
        #       Code can be string or array (maybe warn if string)
        #       There can be a comment
        #       There shouldn't be any other fields
        print(errors)
        print("I'm done!")<commit_msg>Add even more checks<commit_after>import os
import json

from django.core.management.base import BaseCommand, CommandError
from web.MetaInfo import MetaInfo

class Command(BaseCommand):
    help = "Validates JSON language files with the meta language files"

    def handle(self, *args, **options):
        errors = []
        warnings = []

        metainfo = MetaInfo()

        # Look though thesaurus directories, see if any files don't match
        # Load MetaInfo, check directories are in MetaInfo
        meta_files = os.listdir("web/thesauruses/_meta")

        for dir in os.listdir("web/thesauruses"):
            if dir == "_meta":
                continue
            path = "web/thesauruses/" + dir
            if os.path.isdir(path):

                if not dir in list(metainfo.languages.values()):
                    errors.append(path + " is not listed as a language in meta_info.json")

                files = os.listdir("web/thesauruses/" + dir)
                for f in files:
                    path = "web/thesauruses/" + dir + "/" + f
                    if not f in meta_files:
                        errors.append(path + " is not a valid concept filename")

        # Also check structures are in MetaInfo

        for meta_file in meta_files:
            structure_name = meta_file[:-5]
            if not structure_name in list(metainfo.data_structures.values()):
                errors.append("web/thesauruses/_meta/" + meta_file + " is not listed as a structure in meta_info.json")

        # Open up thesaurus directory
        language_dirs = os.listdir("web/thesauruses/")
        for lang_dir in language_dirs:
            if lang_dir == "_meta":
                continue
            if os.path.isfile("web/thesauruses/" + lang_dir):
                continue

        #    Open up each structures file
            structure_files = os.listdir("web/thesauruses/" + lang_dir)
            for structure_file in structure_files:
                structure = structure_file[:-5]
                metastructure = metainfo.structure(structure)

        #       Ensure valid lang/version/name
                meta_structure_file_path = os.path.join(
                    "web", "thesauruses", lang_dir, structure) + ".json"
                with open(meta_structure_file_path, 'r') as meta_structure_file:
                    raw_file_data = meta_structure_file.read()
                    # parse file
                    meta_structure_file_json = json.loads(raw_file_data)

                    language = meta_structure_file_json["meta"]["language"]
                    language_version = meta_structure_file_json["meta"]["language_version"]
                    language_name = meta_structure_file_json["meta"]["language_name"]
                    if not language:
                        errors.append(lang_dir + "/" + structure + ".json has an empty language attribute and needs to be updated")
                    elif language == "language_id":
                        errors.append(lang_dir + "/" + structure + ".json has the default language attribute and needs to be updated")
                    elif not language == lang_dir:
                        errors.append(lang_dir + "/" + structure + ".json has a language attribute that should be '" + lang_dir + "' and needs to be updated")

                    if not language_version:
                        errors.append(lang_dir + "/" + structure + ".json has an empty language_version attribute and needs to be updated")
                    elif language_version == "version.number":
                        errors.append(lang_dir + "/" + structure + ".json has the default language_version attribute and needs to be updated")

                    if not language_name:
                        errors.append(lang_dir + "/" + structure + ".json has an empty language_name attribute and needs to be updated")
                    elif language_name == "Human-Friendly Language Name":
                        errors.append(lang_dir + "/" + structure + ".json has the default language_name attribute and needs to be updated")

        #       Ensure categories aren't in file
                # This works, uncomment to turn back on
        #             if "categories" in meta_structure_file_json:
        #                 errors.append(lang_dir + "/" + structure + ".json has categories in it, which are no longer needed in language files")

        #       Ensure name lines are removed
                    for item in meta_structure_file_json[structure]:
                        structure_item_data = meta_structure_file_json[structure][item]

                        # if "name" in structure_item_data:
                        #     errors.append(
                        #         lang_dir + "/" + structure + ".json, ID:" + item + " has a name line that can be removed")

        #       Ensure there's either code or not-implemented
                        has_code = "code" in structure_item_data
                        has_not_implemented = "not-implemented" in structure_item_data
                        has_comment = "comment" in structure_item_data
                        if (has_code and has_not_implemented):
                            errors.append(
                                lang_dir + "/" + structure + ".json, ID:" + item + " should have 'code' or 'not-implemented', not both")
                        elif (not has_code and not has_not_implemented):
                            errors.append(
                                lang_dir + "/" + structure + ".json, ID:" + item + " is missing a needed 'code' or 'not-implemented' line")
        #       Ensure if not-implemented, there's no code line
                        elif has_not_implemented and structure_item_data["not-implemented"] is True and has_code:
                            errors.append(
                                lang_dir + "/" + structure + ".json, ID:" + item +
                                "is not implemented, but has a 'code' line that should be removed")
        #       Ensure if code, there's no not-implemented
                        elif has_code and not structure_item_data["code"] and not has_not_implemented:
                            errors.append(
                                lang_dir + "/" + structure + ".json, ID:" + item +
                                " has an empty 'code' line but doesn't have 'not-implemented' instead")
        #       Code can be string or array (maybe warn if string)
                        elif has_code and isinstance(structure_item_data["code"], str):
                            warnings.append(
                                lang_dir + "/" + structure + ".json, ID:" + item +
                                " has a 'code' line that's a string and could be an array")
        #       There can be a comment
                # nothing to check, so no code
        #       There shouldn't be any other fields
                        for key in structure_item_data:
                            if not (key == "code" or key == "comment" or key == "not-implemented"):
                                warnings.append(
                                    lang_dir + "/" + structure + ".json, ID:" + item +
                                    " has a line '" + key + "' that's unknown")

        print(errors)
        print("I'm done!")
{'fraction_non_alphanumeric': 0.08443736374308235, 'fraction_numerical': 0.000335401643468053, 'mean_word_length': 1.6604952041043943, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '245771', 'n_tokens_mistral': 2995, 'n_tokens_neox': 2879, 'n_words': 927}