<commit_before>#pylint: disable=missing-docstring
#* This file is part of the MOOSE framework
#* https://www.mooseframework.org
#*
#* All rights reserved, see COPYRIGHT for full restrictions
#* https://github.com/idaholab/moose/blob/master/COPYRIGHT
#*
#* Licensed under LGPL 2.1, please see LICENSE for details
#* https://www.gnu.org/licenses/lgpl-2.1.html
#pylint: enable=missing-docstring
import os
import sys
import re
import subprocess
import mooseutils
import MooseDocs

try:
    import bs4
except ImportError:
    print "The BeutifulSoup package (bs4) is required for the verify command, " \
          "it may be downloaded by running the following:\n" \
          "    pip install --user bs4"
    sys.exit(1)

def command_line_options(subparser, parent):
    """Command line options for 'verify' command."""
    parser = subparser.add_parser('verify', parents=[parent], help="Testing only, do not use.")
    parser.add_argument('-f', '--form', default='materialize',
                        choices=['materialize', 'html', 'json', 'latex'],
                        help="The desired output format to verify.")
    parser.add_argument('--update-gold', action='store_true',
                        help="Copy the rendered results to the gold directory.")
    parser.add_argument('--executioner',
                        default='MooseDocs.base.ParallelBarrier',
                        help="Select the mode of execution.")

def insert_moose_dir(content):
    """Helper for adding '${MOOSE_DIR}' to content."""
    return content.replace(os.getenv('MOOSE_DIR'), '${MOOSE_DIR}')

def replace_uuid4(content):
    """Replace uuid.uuid4() numbers."""
    return re.sub(r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}',
                  r'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX', content)

def update_gold_helper(gold, out_content):
    """Update the gold files."""
    dirname = os.path.dirname(gold)
    if not os.path.isdir(dirname):
        os.makedirs(dirname)
    with open(gold, 'w') as fid:
        print "WRITING GOLD: {}".format(gold)
        fid.write(out_content)

def compare(out_fname, out_dir, gold_dir, update_gold=False):
    """Compare supplied file with gold counter part."""

    errno = 0
    gold_fname = out_fname.replace(out_dir, gold_dir)

    # Read the content to be tested
    with open(out_fname, 'r') as fid:
        out_content = insert_moose_dir(replace_uuid4(fid.read()))

    # Update gold content
    if update_gold:
        update_gold_helper(gold_fname, out_content)

    if not os.path.exists(gold_fname):
        print mooseutils.colorText('MISSING GOLD: {}'.format(gold_fname), 'RED')
        errno = 1
    else:
        with open(gold_fname, 'r') as fid:<commit_msg>Update verify to allow for package filename to be updated<commit_after>#pylint: disable=missing-docstring
#* This file is part of the MOOSE framework
#* https://www.mooseframework.org
#*
#* All rights reserved, see COPYRIGHT for full restrictions
#* https://github.com/idaholab/moose/blob/master/COPYRIGHT
#*
#* Licensed under LGPL 2.1, please see LICENSE for details
#* https://www.gnu.org/licenses/lgpl-2.1.html
#pylint: enable=missing-docstring
import os
import sys
import re
import subprocess
import mooseutils
import MooseDocs

try:
    import bs4
except ImportError:
    print "The BeutifulSoup package (bs4) is required for the verify command, " \
          "it may be downloaded by running the following:\n" \
          "    pip install --user bs4"
    sys.exit(1)

def command_line_options(subparser, parent):
    """Command line options for 'verify' command."""
    parser = subparser.add_parser('verify', parents=[parent], help="Testing only, do not use.")
    parser.add_argument('-f', '--form', default='materialize',
                        choices=['materialize', 'html', 'json', 'latex'],
                        help="The desired output format to verify.")
    parser.add_argument('--update-gold', action='store_true',
                        help="Copy the rendered results to the gold directory.")
    parser.add_argument('--executioner',
                        default='MooseDocs.base.ParallelBarrier',
                        help="Select the mode of execution.")

def prepare_content(content):
    """Performs necessary substitutions for consistent comparisions."""
    content = insert_moose_dir(content)
    content = replace_uuid4(content)
    content = replace_package_file(content)
    return content

def insert_moose_dir(content):
    """Helper for adding '${MOOSE_DIR}' to content."""
    return content.replace(os.getenv('MOOSE_DIR'), '${MOOSE_DIR}')

def replace_uuid4(content):
    """Replace uuid.uuid4() numbers."""
    return re.sub(r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}',
                  r'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX', content)

def replace_package_file(content):
    """Replace the package filename with something consistent."""
    return re.sub(r'(moose-environment\S*\.pkg)', r'moose-environment.pkg', content)

def update_gold_helper(gold, out_content):
    """Update the gold files."""
    dirname = os.path.dirname(gold)
    if not os.path.isdir(dirname):
        os.makedirs(dirname)
    with open(gold, 'w') as fid:
        print "WRITING GOLD: {}".format(gold)
        fid.write(out_content)

def compare(out_fname, out_dir, gold_dir, update_gold=False):
    """Compare supplied file with gold counter part."""

    errno = 0
    gold_fname = out_fname.replace(out_dir, gold_dir)

    # Read the content to be tested
    with open(out_fname, 'r') as fid:
        out_content = prepare_content(fid.read())

    # Update gold content
    if update_gold:
        update_gold_helper(gold_fname, out_content)

    if not os.path.exists(gold_fname):
        print mooseutils.colorText('MISSING GOLD: {}'.format(gold_fname), 'RED')
        errno = 1
    else:
        with open(gold_fname, 'r') as fid:
{'fraction_non_alphanumeric': 0.13300576075906473, 'fraction_numerical': 0.009827177228058285, 'mean_word_length': 3.2254831782390836, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1347992', 'n_tokens_mistral': 1877, 'n_tokens_neox': 1848, 'n_words': 469}