<commit_before>import json
import os
import subprocess
import yaml
from jsonschema.exceptions import ValidationError
from buildtest.defaults import REPO_FILE, BUILDSPEC_CACHE_FILE
from buildtest.config import get_default_settings
from buildtest.utils.file import is_file, is_dir, walk_tree
from buildtest.buildsystem.base import BuildspecParser


def func_buildspec_find(args):
    """Entry point for ``buildtest buildspec find``. This method
       will attempt to read for buildspec cache file (BUILDSPEC_CACHE_FILE)
       if found and print a list of all buildspecs. Otherwise, it will
       read the repo file (REPO_FILE) and find all buildspecs and validate
       them via BuildspecParser. BuildspecParser will raise SystemError or
       ValidationError if a buildspec is invalid which will be added to list
       of invalid buildspecs. Finally we print a list of all valid buildspecs
       and any invalid buildspecs are written to file along with error
       message.

       Parameters

       :param args: Input argument from command line passed from argparse
       :return: A list of valid buildspecs found in all repositories.
    """

    cache = {}
    # implements buildtest buildspec find --clear which removes cache file before finding all buildspecs
    if args.clear:
        try:
            os.remove(BUILDSPEC_CACHE_FILE)
            print(f"Clearing cache file: {BUILDSPEC_CACHE_FILE}")
        except OSError:
            pass

    if is_file(BUILDSPEC_CACHE_FILE):
        with open(BUILDSPEC_CACHE_FILE, "r") as fd:
            cache = json.loads(fd.read())
    # if cache file is not found, then we will build cache by searching
    # all buildspecs based on available repos found in REPO_FILE and
    # traverse directory to find all .yml files
    else:
        # if repo file not found then terminate with message since we can't find
        # any buildspecs
        if not is_file(REPO_FILE):
            raise SystemExit(
                "Unable to find any buildspecs because no repositories found."
            )

        with open(REPO_FILE, "r") as fd:
            repo_dict = yaml.load(fd.read(), Loader=yaml.SafeLoader)

        paths = []
        for repo in repo_dict.keys():
            # only add if repository cloned locally exists in filepath
            if is_dir(repo_dict[repo]["dest"]):
                paths.append(repo_dict[repo]["dest"])

        buildspecs = []
        invalid_buildspecs = {}
        parse = None
        # add all buildspecs from each repo. walk_tree will find all .yml files
        # recursively and add them to list
        for path in paths:
            buildspec = walk_tree(path, ".yml")
            buildspecs += buildspec

        # process each buildspec by invoking BuildspecParser which will validate
        # buildspec, if it fails it will raise SystemExit or ValidationError in
        # this case we skip to next buildspec
        for buildspec in buildspecs:

            try:
                parse = BuildspecParser(buildspec)
            # any buildspec that raises SystemExit or ValidationError imply
            # buildspec is not valid, we add this to invalid list along with
            # error message and skip to next buildspec
            except (SystemExit, ValidationError) as err:
                invalid_buildspecs[buildspec] = err
                continue

            recipe = parse.recipe

            cache[buildspec] = {}
            cache[buildspec]["sections"] = []
            cache[buildspec]["section_summary"] = {}

            for name in recipe.keys():

                if not isinstance(recipe[name], dict):
                    continue

                cache[buildspec]["sections"].append(name)
                cache[buildspec]["section_summary"][name] = {}<commit_msg>fix bug in buildtest buildspec find where names of tests were not picked (#337)<commit_after>import json
import os
import subprocess
import yaml
from jsonschema.exceptions import ValidationError
from buildtest.defaults import REPO_FILE, BUILDSPEC_CACHE_FILE
from buildtest.config import get_default_settings
from buildtest.utils.file import is_file, is_dir, walk_tree
from buildtest.buildsystem.base import BuildspecParser


def func_buildspec_find(args):
    """Entry point for ``buildtest buildspec find``. This method
       will attempt to read for buildspec cache file (BUILDSPEC_CACHE_FILE)
       if found and print a list of all buildspecs. Otherwise, it will
       read the repo file (REPO_FILE) and find all buildspecs and validate
       them via BuildspecParser. BuildspecParser will raise SystemError or
       ValidationError if a buildspec is invalid which will be added to list
       of invalid buildspecs. Finally we print a list of all valid buildspecs
       and any invalid buildspecs are written to file along with error
       message.

       Parameters

       :param args: Input argument from command line passed from argparse
       :return: A list of valid buildspecs found in all repositories.
    """

    cache = {}
    # implements buildtest buildspec find --clear which removes cache file before finding all buildspecs
    if args.clear:
        try:
            os.remove(BUILDSPEC_CACHE_FILE)
            print(f"Clearing cache file: {BUILDSPEC_CACHE_FILE}")
        except OSError:
            pass

    if is_file(BUILDSPEC_CACHE_FILE):
        with open(BUILDSPEC_CACHE_FILE, "r") as fd:
            cache = json.loads(fd.read())
    # if cache file is not found, then we will build cache by searching
    # all buildspecs based on available repos found in REPO_FILE and
    # traverse directory to find all .yml files
    else:
        # if repo file not found then terminate with message since we can't find
        # any buildspecs
        if not is_file(REPO_FILE):
            raise SystemExit(
                "Unable to find any buildspecs because no repositories found."
            )

        with open(REPO_FILE, "r") as fd:
            repo_dict = yaml.load(fd.read(), Loader=yaml.SafeLoader)

        paths = []
        for repo in repo_dict.keys():
            # only add if repository cloned locally exists in filepath
            if is_dir(repo_dict[repo]["dest"]):
                paths.append(repo_dict[repo]["dest"])

        buildspecs = []
        invalid_buildspecs = {}
        parse = None
        # add all buildspecs from each repo. walk_tree will find all .yml files
        # recursively and add them to list
        for path in paths:
            buildspec = walk_tree(path, ".yml")
            buildspecs += buildspec

        # process each buildspec by invoking BuildspecParser which will validate
        # buildspec, if it fails it will raise SystemExit or ValidationError in
        # this case we skip to next buildspec
        for buildspec in buildspecs:

            try:
                parse = BuildspecParser(buildspec)
            # any buildspec that raises SystemExit or ValidationError imply
            # buildspec is not valid, we add this to invalid list along with
            # error message and skip to next buildspec
            except (SystemExit, ValidationError) as err:
                invalid_buildspecs[buildspec] = err
                continue

            recipe = parse.recipe["buildspecs"]

            cache[buildspec] = {}
            cache[buildspec]["sections"] = []
            cache[buildspec]["section_summary"] = {}

            for name in recipe.keys():

                if not isinstance(recipe[name], dict):
                    continue

                cache[buildspec]["sections"].append(name)
                cache[buildspec]["section_summary"][name] = {}
{'fraction_non_alphanumeric': 0.06147220046985121, 'fraction_numerical': 0.00039154267815191856, 'mean_word_length': 2.3698328935795954, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3799494', 'n_tokens_mistral': 2036, 'n_tokens_neox': 1933, 'n_words': 759}