<commit_before>"""
Module containing filter functions that allow code to be highlighted
from within Jinja templates.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2013, the IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

# pygments must not be imported at the module level
# because errors should be raised at runtime if it's actually needed,
# not import time, when it may not be needed.

# Our own imports
from IPython.nbconvert.utils.base import NbConvertBase

#-----------------------------------------------------------------------------
# Globals and constants
#-----------------------------------------------------------------------------

MULTILINE_OUTPUTS = ['text', 'html', 'svg', 'latex', 'javascript', 'json']

#-----------------------------------------------------------------------------
# Utility functions
#-----------------------------------------------------------------------------

__all__ = [
    'Highlight2HTML',
    'Highlight2Latex'
]


class Highlight2HTML(NbConvertBase):

    def __call__(self, source, language=None, metadata=None):
        """
        Return a syntax-highlighted version of the input source as html output.

        Parameters
        ----------
        source : str
            source of the cell to highlight
        language : str
            language to highlight the syntax of
        metadata : NotebookNode cell metadata
            metadata of the cell to highlight
        """
        from pygments.formatters import HtmlFormatter
        if not language:
            language=self.default_language

        return _pygments_highlight(source if len(source) > 0 else ' ', HtmlFormatter(), language, metadata)


class Highlight2Latex(NbConvertBase):

    def __call__(self, source, language=None, metadata=None, strip_verbatim=False):
        """
        Return a syntax-highlighted version of the input source as latex output.

        Parameters
        ----------
        source : str
            source of the cell to highlight
        language : str
            language to highlight the syntax of
        metadata : NotebookNode cell metadata
            metadata of the cell to highlight
        strip_verbatim : bool
            remove the Verbatim environment that pygments provides by default
        """
        from pygments.formatters import LatexFormatter
        if not language:
            language=self.default_language

        latex = _pygments_highlight(source, LatexFormatter(), language, metadata)
        if strip_verbatim:
            latex = latex.replace(r'\begin{Verbatim}[commandchars=\\\{\}]' + '\n', '')
            return latex.replace('\n\\end{Verbatim}\n', '')<commit_msg>Pygment now adds a language-class to the highlighted html so that postprocessors can use it to identify the code block and the language of the code inside.<commit_after>"""
Module containing filter functions that allow code to be highlighted
from within Jinja templates.
"""
#-----------------------------------------------------------------------------
# Copyright (c) the IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

# pygments must not be imported at the module level
# because errors should be raised at runtime if it's actually needed,
# not import time, when it may not be needed.

# Our own imports
from IPython.nbconvert.utils.base import NbConvertBase

#-----------------------------------------------------------------------------
# Globals and constants
#-----------------------------------------------------------------------------

MULTILINE_OUTPUTS = ['text', 'html', 'svg', 'latex', 'javascript', 'json']

#-----------------------------------------------------------------------------
# Utility functions
#-----------------------------------------------------------------------------

__all__ = [
    'Highlight2HTML',
    'Highlight2Latex'
]


class Highlight2HTML(NbConvertBase):

    def __call__(self, source, language=None, metadata=None):
        """
        Return a syntax-highlighted version of the input source as html output.

        Parameters
        ----------
        source : str
            source of the cell to highlight
        language : str
            language to highlight the syntax of
        metadata : NotebookNode cell metadata
            metadata of the cell to highlight
        """
        from pygments.formatters import HtmlFormatter
        if not language:
            language=self.default_language

        return _pygments_highlight(source if len(source) > 0 else ' ',
                                   # needed to help post processors:
                                   HtmlFormatter(cssclass="hl-"+language), 
                                   language, metadata)


class Highlight2Latex(NbConvertBase):

    def __call__(self, source, language=None, metadata=None, strip_verbatim=False):
        """
        Return a syntax-highlighted version of the input source as latex output.

        Parameters
        ----------
        source : str
            source of the cell to highlight
        language : str
            language to highlight the syntax of
        metadata : NotebookNode cell metadata
            metadata of the cell to highlight
        strip_verbatim : bool
            remove the Verbatim environment that pygments provides by default
        """
        from pygments.formatters import LatexFormatter
        if not language:
            language=self.default_language

        latex = _pygments_highlight(source, LatexFormatter(), language, metadata)
        if strip_verbatim:
            latex = latex.replace(r'\begin{Verbatim}[commandchars=\\\{\}]' + '\n', '')
            return latex.replace('\n\\end{Verbatim}\n', '')
{'fraction_non_alphanumeric': 0.25622775800711745, 'fraction_numerical': 0.0021661766981278044, 'mean_word_length': 1.3565439300036457, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '509869', 'n_tokens_mistral': 1394, 'n_tokens_neox': 1310, 'n_words': 498}