<commit_before>"""Developer tools for MooseDocs."""
import os
import collections
import logging

import anytree

import MooseDocs
from MooseDocs import common
from MooseDocs.tree import syntax
from MooseDocs.common import exceptions

LOG = logging.getLogger(__name__)
STUB_HEADER = '<!-- MOOSE Documentation Stub: Remove this when content is added. -->'

def command_line_options(subparser, parent):
    """Define the 'check' command."""

    parser = subparser.add_parser('check',
                                  parents=[parent],
                                  help='Syntax checking tools for MooseDocs.')

    parser.add_argument('--config', default='config.yml', help="The configuration file.")
    parser.add_argument('--generate', action='store_true',
                        help="When checking the application for complete documentation generate "
                             "any missing markdown documentation files.")
    parser.add_argument('--update', action='store_true',
                        help="When checking the application for complete documentation generate "
                             "any missing markdown documentation files and update the stubs for "
                             "files that have not been modified.")
    parser.add_argument('--dump', action='store_true',
                        help="Dump the complete MooseDocs syntax tree to the screen.")
    parser.add_argument('--object_prefix',
                        default=os.path.join('doc', 'content', 'source'),
                        help="The folder install prefix for MooseObject documentations pages.")
    parser.add_argument('--syntax_prefix',
                        default=os.path.join('doc', 'content', 'syntax'),
                        help="The folder install prefix for MooseObject documentations pages.")

def main(options):
    """./moosedocs check"""

    translator, _ = common.load_config(options.config)
    check(translator,
          dump=options.dump,
          update=options.update,
          generate=options.generate,
          object_prefix=options.object_prefix,
          syntax_prefix=options.syntax_prefix)

def check(translator,
          dump=False,
          update=False,
          generate=False,
          object_prefix=os.path.join('doc', 'content', 'source'),
          syntax_prefix=os.path.join('doc', 'content', 'syntax')):
    """Helper to all both main and build.py:main to perform check."""

    # Extract the syntax root node
    app_syntax = None
    extension = None
    for ext in translator.extensions:
        extension = ext
        if isinstance(ext, MooseDocs.extensions.appsyntax.AppSyntaxExtension):
            app_syntax = ext.syntax
            break

    if extension['disable']:
        LOG.info("Syntax is disabled, skipping the check.")
        return

    if app_syntax is None:
        msg = "Failed to locate AppSyntaxExtension for the given configuration."
        raise exceptions.MooseDocsException(msg)

    # Dump the complete syntax for the application
    if dump:
        print app_syntax

    # Perform check for all the nodes
    for node in anytree.PreOrderIter(app_syntax):
        if node.is_root or node.removed:# or node.hidden:# or node.test:
            continue
        elif isinstance(node, syntax.ObjectNode):
            _check_object_node(node, generate, update, object_prefix)
        elif isinstance(node, syntax.SyntaxNode):
            _check_syntax_node(node, generate, update, syntax_prefix)
        else:
            LOG.critical("Unexpected object type of %s, only %s and %s based types are supported",
                         type(node).__name__,
                         syntax.ObjectNode.__name__,
                         syntax.SyntaxNode.__name__)

def _check_object_node(node, generate, update, prefix):
    """
    Check that required pages for supplied ObjectNode (i.e., MooseObject/Action).
    """
    idx = node.source().find('/src/')
    filename = os.path.join(node.source()[:idx], prefix, node.markdown())

    not_exist = not os.path.isfile(filename)
    if not_exist and (not generate) and (not node.hidden):
        msg = "No documentation for %s.\n"
        msg += "    - The page should be located at %s.\n"
        msg += "    - It is possible to generate stub pages using " \
               "'./moosedocs.py check --generate'."
        LOG.error(msg, node.fullpath, filename)
<commit_msg>Handle error in syntax node markdown method when running check<commit_after>"""Developer tools for MooseDocs."""
import os
import collections
import logging

import anytree

import MooseDocs
from MooseDocs import common
from MooseDocs.tree import syntax
from MooseDocs.common import exceptions

LOG = logging.getLogger(__name__)
STUB_HEADER = '<!-- MOOSE Documentation Stub: Remove this when content is added. -->'

def command_line_options(subparser, parent):
    """Define the 'check' command."""

    parser = subparser.add_parser('check',
                                  parents=[parent],
                                  help='Syntax checking tools for MooseDocs.')

    parser.add_argument('--config', default='config.yml', help="The configuration file.")
    parser.add_argument('--generate', action='store_true',
                        help="When checking the application for complete documentation generate "
                             "any missing markdown documentation files.")
    parser.add_argument('--update', action='store_true',
                        help="When checking the application for complete documentation generate "
                             "any missing markdown documentation files and update the stubs for "
                             "files that have not been modified.")
    parser.add_argument('--dump', action='store_true',
                        help="Dump the complete MooseDocs syntax tree to the screen.")
    parser.add_argument('--object_prefix',
                        default=os.path.join('doc', 'content', 'source'),
                        help="The folder install prefix for MooseObject documentations pages.")
    parser.add_argument('--syntax_prefix',
                        default=os.path.join('doc', 'content', 'syntax'),
                        help="The folder install prefix for MooseObject documentations pages.")

def main(options):
    """./moosedocs check"""

    translator, _ = common.load_config(options.config)
    check(translator,
          dump=options.dump,
          update=options.update,
          generate=options.generate,
          object_prefix=options.object_prefix,
          syntax_prefix=options.syntax_prefix)

def check(translator,
          dump=False,
          update=False,
          generate=False,
          object_prefix=os.path.join('doc', 'content', 'source'),
          syntax_prefix=os.path.join('doc', 'content', 'syntax')):
    """Helper to all both main and build.py:main to perform check."""

    # Extract the syntax root node
    app_syntax = None
    extension = None
    for ext in translator.extensions:
        extension = ext
        if isinstance(ext, MooseDocs.extensions.appsyntax.AppSyntaxExtension):
            app_syntax = ext.syntax
            break

    if extension['disable']:
        LOG.info("Syntax is disabled, skipping the check.")
        return

    if app_syntax is None:
        msg = "Failed to locate AppSyntaxExtension for the given configuration."
        raise exceptions.MooseDocsException(msg)

    # Dump the complete syntax for the application
    if dump:
        print app_syntax

    # Perform check for all the nodes
    for node in anytree.PreOrderIter(app_syntax):
        if node.is_root or node.removed:# or node.hidden:# or node.test:
            continue
        elif isinstance(node, syntax.ObjectNode):
            _check_object_node(node, generate, update, object_prefix)
        elif isinstance(node, syntax.SyntaxNode):
            _check_syntax_node(node, generate, update, syntax_prefix)
        else:
            LOG.critical("Unexpected object type of %s, only %s and %s based types are supported",
                         type(node).__name__,
                         syntax.ObjectNode.__name__,
                         syntax.SyntaxNode.__name__)

def _check_object_node(node, generate, update, prefix):
    """
    Check that required pages for supplied ObjectNode (i.e., MooseObject/Action).
    """
    idx = node.source().find('/src/')

    # If the markdown method returns None, it failed to locate the file, which throws an error
    md = node.markdown()
    if md is None:
        return
    filename = os.path.join(node.source()[:idx], prefix, md)

    not_exist = not os.path.isfile(filename)
    if not_exist and (not generate) and (not node.hidden):
        msg = "No documentation for %s.\n"
        msg += "    - The page should be located at %s.\n"
        msg += "    - It is possible to generate stub pages using " \
               "'./moosedocs.py check --generate'."
        LOG.error(msg, node.fullpath, filename)

{'fraction_non_alphanumeric': 0.09885720625762787, 'fraction_numerical': 0.0, 'mean_word_length': 2.355919583023083, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5696732', 'n_tokens_mistral': 2340, 'n_tokens_neox': 2250, 'n_words': 696}