<commit_before>import os
from mooseutils import colorText, git_ls_files, git_root_dir
try:
    from hit_load import hit_load
except:
    raise ImportError('hit package failed to load, but it is required.')

class SQAStats(object):
    """Data wrapper for SQA statistic information."""
    def __init__(self, title):
        self.title = title
        self.files = 0
        self.files_with_requirement = 0
        self.tests = 0
        self.tests_with_requirement = 0
        self.tests_deprecated = 0

    @property
    def complete(self):
        return float(self.tests_with_requirement)/float(self.tests - self.tests_deprecated) if self.tests else 0

    def __iadd__(self, other):
        self.files += other.files
        self.files_with_requirement += other.files_with_requirement
        self.tests += other.tests
        self.tests_with_requirement += other.tests_with_requirement
        return self

    def __str__(self):
        """Display requirement stats."""
        out = []
        out.append(colorText(self.title, 'LIGHT_YELLOW'))
        out.append('                   Complete: {:2.1f}%'.format(self.complete*100))
        out.append('      Total Number of Files: {}'.format(self.files))
        out.append('    Files with Requirements: {}'.format(self.files_with_requirement))
        out.append('      Total Number of Tests: {}'.format(self.tests))
        out.append('    Tests with Requirements: {}'.format(self.tests_with_requirement))
        out.append('           Deprecated Tests: {}'.format(self.tests_deprecated))
        out.append('            Tests Remaining: {}'.format(self.tests - self.tests_deprecated - self.tests_with_requirement))
        return '\n'.join(out)

def compute_requirement_stats(location, specs=['tests'], working_dir=None, show=True, list_missing=False):
    """
    Report requirement statistics for the test spec files with the supplied location.

    Inputs:
        location: Path to directory contain test specifications, the supplied path should be
                  relative to the cwd input.
        specs: The filename(s) to consider
        working_dir: The working directory, if not supplied the root directory of the repository is used

    """
    files_with_missing_requirements = set()
    working_dir = git_root_dir(os.getcwd()) if working_dir is None else working_dir
    data = SQAStats(location)
    location = os.path.join(working_dir, location)
    for filename in git_ls_files(location):
        if not os.path.isfile(filename):
            continue
        fname = os.path.basename(filename)
        if fname in specs:
            root = hit_load(filename)
            has_requirement = False
            for child in root.children[0]:
                data.tests += 1

                if root.children[0].get('deprecated', False) or child.get('deprecated', False):
                    data.tests_deprecated += 1

                if child.get('requirement', None):
                    has_requirement = True
                    data.tests_with_requirement += 1

            data.files += 1
            if has_requirement:
                data.files_with_requirement += 1
            else:
                files_with_missing_requirements.add(filename)

    if show:
        print(data)
    if list_missing:
        print('\n'.join(files_with_missing_requirements))
    return data<commit_msg>Improve missing requirement listing<commit_after>import os
from mooseutils import colorText, git_ls_files, git_root_dir
try:
    from hit_load import hit_load
except:
    raise ImportError('hit package failed to load, but it is required.')

class SQAStats(object):
    """Data wrapper for SQA statistic information."""
    def __init__(self, title):
        self.title = title
        self.files = 0
        self.files_with_requirement = 0
        self.tests = 0
        self.tests_with_requirement = 0
        self.tests_deprecated = 0

    @property
    def complete(self):
        return float(self.tests_with_requirement)/float(self.tests - self.tests_deprecated) if self.tests else 0

    def __iadd__(self, other):
        self.files += other.files
        self.files_with_requirement += other.files_with_requirement
        self.tests += other.tests
        self.tests_with_requirement += other.tests_with_requirement
        return self

    def __str__(self):
        """Display requirement stats."""
        out = []
        out.append(colorText(self.title, 'LIGHT_YELLOW'))
        out.append('                   Complete: {:2.1f}%'.format(self.complete*100))
        out.append('      Total Number of Files: {}'.format(self.files))
        out.append('    Files with Requirements: {}'.format(self.files_with_requirement))
        out.append('      Total Number of Tests: {}'.format(self.tests))
        out.append('    Tests with Requirements: {}'.format(self.tests_with_requirement))
        out.append('           Deprecated Tests: {}'.format(self.tests_deprecated))
        out.append('            Tests Remaining: {}'.format(self.tests - self.tests_deprecated - self.tests_with_requirement))
        return '\n'.join(out)

def compute_requirement_stats(location, specs=['tests'], working_dir=None, show=True, list_missing=False):
    """
    Report requirement statistics for the test spec files with the supplied location.

    Inputs:
        location: Path to directory contain test specifications, the supplied path should be
                  relative to the cwd input.
        specs: The filename(s) to consider
        working_dir: The working directory, if not supplied the root directory of the repository is used

    """
    tests_with_missing_requirements = set()
    working_dir = git_root_dir(os.getcwd()) if working_dir is None else working_dir
    data = SQAStats(location)
    location = os.path.join(working_dir, location)
    for filename in git_ls_files(location):
        if not os.path.isfile(filename):
            continue
        fname = os.path.basename(filename)
        if fname in specs:
            root = hit_load(filename)
            has_requirement = False
            for child in root.children[0]:
                data.tests += 1

                deprecated = root.children[0].get('deprecated', False) or child.get('deprecated', False)
                if deprecated:
                    data.tests_deprecated += 1

                if child.get('requirement', None):
                    has_requirement = True
                    data.tests_with_requirement += 1
                elif not deprecated:
                    tests_with_missing_requirements.add((filename, child.name))

            data.files += 1

    if show:
        print(data)
    if list_missing and tests_with_missing_requirements:
        print('\nMissing Requirements:')
        for filename, test in tests_with_missing_requirements:
            print('{}:{}'.format(filename, test))

    return data
{'fraction_non_alphanumeric': 0.09490639965171964, 'fraction_numerical': 0.005079088666376433, 'mean_word_length': 2.456369107321966, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4034372', 'n_tokens_mistral': 1927, 'n_tokens_neox': 1847, 'n_words': 495}