<commit_before>#pylint: disable=missing-docstring
import os
import re

import mooseutils

import MooseDocs
from MooseDocs import common
from MooseDocs.common import exceptions
from MooseDocs.extensions import command, floats
from MooseDocs.tree import tokens

def make_extension(**kwargs):
    return ListingExtension(**kwargs)

class ListingExtension(command.CommandExtension):
    """
    Provides !listing command for including source code from files.
    """

    @staticmethod
    def defaultConfig():
        config = command.CommandExtension.defaultConfig()
        config['prefix'] = (u'Listing', "The caption prefix (e.g., Fig.).")
        return config

    def extend(self, reader, renderer):

        self.requires(command, floats)
        self.addCommand(LocalListingCommand())
        self.addCommand(FileListingCommand())
        self.addCommand(InputListingCommand())

class LocalListingCommand(command.CommandComponent):
    COMMAND = 'listing'
    SUBCOMMAND = None

    @staticmethod
    def defaultSettings():
        settings = command.CommandComponent.defaultSettings()
        settings['caption'] = (None, "The caption to use for the listing content.")
        settings['prefix'] = (None, "Text to include prior to the included text.")
        settings['max-height'] = (u'350px', "The default height for listing content.")
        settings['language'] = (None, "The language to use for highlighting, if not supplied it " \
                                      "will be inferred from the extension (if possible).")
        return settings

    def createToken(self, info, parent):
        flt = floats.Float(parent)
        content = info['inline'] if 'inline' in info else info['block']
        self.addCaption(flt)
        tokens.Code(flt, style="max-height:{};".format(self.settings['max-height']),
                    language=self.settings['language'], code=content)
        return parent

    # TODO: this is repeated in media, table, make a helper...(CaptionMixin)
    def addCaption(self, flt):

        cap = self.settings['caption']
        key = self.settings['id']
        if key:
            if self.settings['prefix'] is not None:
                prefix = self.settings['prefix']
            else:
                prefix = self.extension.get('prefix')
            caption = floats.Caption(flt, key=key, prefix=prefix)
            if cap:
                self.translator.reader.parse(caption, cap, MooseDocs.INLINE)
        elif cap:
            caption = floats.Caption(flt)
            self.translator.reader.parse(caption, cap, MooseDocs.INLINE)

class FileListingCommand(LocalListingCommand):
    COMMAND = 'listing'
    SUBCOMMAND = '*'

    @staticmethod
    def defaultSettings():
        settings = LocalListingCommand.defaultSettings()
        settings['caption'] = (None, "The caption to use for the listing content.")
        settings['link'] = (True, "Include a link to the filename after the listing.")
        settings['header'] = (None, "Text to include prior to the included text.")
        settings['footer'] = ('', "Text to include after to the included text.")
        settings['indent'] = (0, "The level of indenting to apply to the included text.")
        settings['strip-header'] = (True, "When True the MOOSE header is removed for display.")
        settings['strip-extra-newlines'] = (True, "Removes extraneous new lines from the text.")
        settings['strip-leading-whitespace'] = (False, "When True leading white-space is removed "
                                                       "from the included text.")
        settings['line'] = (None, "A portion of text that unique identifies a single line to "
                                  "include.")
        settings['re'] = (False, "Extract content via a regex, if the 'content' group exists it " \
                                 "is used as the desired content, otherwise group 0 is used.")
        settings['re-flags'] = ('re.M|re.S|re.U', "Python re flags.")
        settings['start'] = (None, "A portion of text that unique identifies the starting "
                                   "location for including text, if not provided the beginning "
                                   "of the file is utilized.")
        settings['end'] = (None, "A portion of text that unique identifies the ending location "
                                 "for including text, if not provided the end of the file is "
                                 "used. By default this line is not included in the display.")
        settings['include-start'] = (True, "When False the texted captured by the 'start' setting "
                                           "is excluded in the displayed text.")
        settings['include-end'] = (False, "When True the texted captured by the 'end' setting is "
                                          "included in the displayed text.")
        return settings

    def createToken(self, info, parent):
        """
        Build the tokens needed for displaying code listing.
        """

        # Locate filename
        filename = common.check_filenames(info['subcommand'])

        # Listing container
        flt = floats.Float(parent)
        self.addCaption(flt)

        # Create code token
        lang = self.settings.get('language')
        lang = lang if lang else common.get_language(filename)
        tokens.Code(flt, style="max-height:{};".format(self.settings['max-height']),
                    code=self.extractContent(filename, self.settings),
                    language=lang)

        # Add bottom modal
        if self.settings['link']:
            code = tokens.Code(None, language=lang, code=common.read(filename))
            floats.ModalLink(flt, url=unicode(filename), bottom=True, content=code,
                             string=u'({})'.format(os.path.relpath(filename, MooseDocs.ROOT_DIR)),
                             title=tokens.String(None, content=unicode(filename)))

        return parent


    def extractContent(self, filename, settings):
        """
        Extract the desired content from the supplied raw text from a file.

        Inputs:
            filename[unicode]: The file to read (known to exist already).
            settings[dict]: The setting from the createToken method.
        """

        content = common.read(filename)
        if settings['re']:
            content = common.regex(self.settings['re'], content, eval(self.settings['re-flags']))

        elif settings['line']:
            content = self.extractLine(content, settings["line"])

        elif settings['start'] or settings['end']:
            content = self.extractLineRange(content,
                                            settings['start'],
                                            settings['end'],
                                            settings['include-start'],
                                            settings['include-end'])
<commit_msg>Remove absolute file paths from listings (refs #12181)<commit_after>#pylint: disable=missing-docstring
import os
import re

import mooseutils

import MooseDocs
from MooseDocs import common
from MooseDocs.common import exceptions
from MooseDocs.extensions import command, floats
from MooseDocs.tree import tokens

def make_extension(**kwargs):
    return ListingExtension(**kwargs)

class ListingExtension(command.CommandExtension):
    """
    Provides !listing command for including source code from files.
    """

    @staticmethod
    def defaultConfig():
        config = command.CommandExtension.defaultConfig()
        config['prefix'] = (u'Listing', "The caption prefix (e.g., Fig.).")
        return config

    def extend(self, reader, renderer):

        self.requires(command, floats)
        self.addCommand(LocalListingCommand())
        self.addCommand(FileListingCommand())
        self.addCommand(InputListingCommand())

class LocalListingCommand(command.CommandComponent):
    COMMAND = 'listing'
    SUBCOMMAND = None

    @staticmethod
    def defaultSettings():
        settings = command.CommandComponent.defaultSettings()
        settings['caption'] = (None, "The caption to use for the listing content.")
        settings['prefix'] = (None, "Text to include prior to the included text.")
        settings['max-height'] = (u'350px', "The default height for listing content.")
        settings['language'] = (None, "The language to use for highlighting, if not supplied it " \
                                      "will be inferred from the extension (if possible).")
        return settings

    def createToken(self, info, parent):
        flt = floats.Float(parent)
        content = info['inline'] if 'inline' in info else info['block']
        self.addCaption(flt)
        tokens.Code(flt, style="max-height:{};".format(self.settings['max-height']),
                    language=self.settings['language'], code=content)
        return parent

    # TODO: this is repeated in media, table, make a helper...(CaptionMixin)
    def addCaption(self, flt):

        cap = self.settings['caption']
        key = self.settings['id']
        if key:
            if self.settings['prefix'] is not None:
                prefix = self.settings['prefix']
            else:
                prefix = self.extension.get('prefix')
            caption = floats.Caption(flt, key=key, prefix=prefix)
            if cap:
                self.translator.reader.parse(caption, cap, MooseDocs.INLINE)
        elif cap:
            caption = floats.Caption(flt)
            self.translator.reader.parse(caption, cap, MooseDocs.INLINE)

class FileListingCommand(LocalListingCommand):
    COMMAND = 'listing'
    SUBCOMMAND = '*'

    @staticmethod
    def defaultSettings():
        settings = LocalListingCommand.defaultSettings()
        settings['caption'] = (None, "The caption to use for the listing content.")
        settings['link'] = (True, "Include a link to the filename after the listing.")
        settings['header'] = (None, "Text to include prior to the included text.")
        settings['footer'] = ('', "Text to include after to the included text.")
        settings['indent'] = (0, "The level of indenting to apply to the included text.")
        settings['strip-header'] = (True, "When True the MOOSE header is removed for display.")
        settings['strip-extra-newlines'] = (True, "Removes extraneous new lines from the text.")
        settings['strip-leading-whitespace'] = (False, "When True leading white-space is removed "
                                                       "from the included text.")
        settings['line'] = (None, "A portion of text that unique identifies a single line to "
                                  "include.")
        settings['re'] = (False, "Extract content via a regex, if the 'content' group exists it " \
                                 "is used as the desired content, otherwise group 0 is used.")
        settings['re-flags'] = ('re.M|re.S|re.U', "Python re flags.")
        settings['start'] = (None, "A portion of text that unique identifies the starting "
                                   "location for including text, if not provided the beginning "
                                   "of the file is utilized.")
        settings['end'] = (None, "A portion of text that unique identifies the ending location "
                                 "for including text, if not provided the end of the file is "
                                 "used. By default this line is not included in the display.")
        settings['include-start'] = (True, "When False the texted captured by the 'start' setting "
                                           "is excluded in the displayed text.")
        settings['include-end'] = (False, "When True the texted captured by the 'end' setting is "
                                          "included in the displayed text.")
        return settings

    def createToken(self, info, parent):
        """
        Build the tokens needed for displaying code listing.
        """

        # Locate filename
        filename = common.check_filenames(info['subcommand'])

        # Listing container
        flt = floats.Float(parent)
        self.addCaption(flt)

        # Create code token
        lang = self.settings.get('language')
        lang = lang if lang else common.get_language(filename)
        tokens.Code(flt, style="max-height:{};".format(self.settings['max-height']),
                    code=self.extractContent(filename, self.settings),
                    language=lang)

        # Add bottom modal
        if self.settings['link']:
            rel_filename = os.path.relpath(filename, MooseDocs.ROOT_DIR)
            code = tokens.Code(None, language=lang, code=common.read(filename))
            floats.ModalLink(flt, url=unicode(rel_filename), bottom=True, content=code,
                             string=u'({})'.format(rel_filename),
                             title=tokens.String(None, content=unicode(filename)))

        return parent


    def extractContent(self, filename, settings):
        """
        Extract the desired content from the supplied raw text from a file.

        Inputs:
            filename[unicode]: The file to read (known to exist already).
            settings[dict]: The setting from the createToken method.
        """

        content = common.read(filename)
        if settings['re']:
            content = common.regex(self.settings['re'], content, eval(self.settings['re-flags']))

        elif settings['line']:
            content = self.extractLine(content, settings["line"])

        elif settings['start'] or settings['end']:
            content = self.extractLineRange(content,
                                            settings['start'],
                                            settings['end'],
                                            settings['include-start'],
                                            settings['include-end'])

{'fraction_non_alphanumeric': 0.10640281064028106, 'fraction_numerical': 0.0010755001075500108, 'mean_word_length': 2.1478221620401716, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 44, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7234275', 'n_tokens_mistral': 3471, 'n_tokens_neox': 3348, 'n_words': 1083}