<commit_before>"""Exporter for the notebook conversion pipeline.

This module defines Exporter, a highly configurable converter
that uses Jinja2 to export notebook files into different format.

You can register both pre-transformers that will act on the notebook format
befor conversion and jinja filter that would then be availlable in the templates
"""

#-----------------------------------------------------------------------------
# Copyright (c) 2013, the IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
from __future__ import print_function, absolute_import

# Stdlib imports
import io
import os
import re

# IPython imports
from IPython.config.configurable import Configurable
from IPython.nbformat import current as nbformat
from IPython.utils.traitlets import MetaHasTraits, Unicode, List, Bool
from IPython.utils.text import indent

# other libs/dependencies
from jinja2 import Environment, FileSystemLoader
from markdown import markdown

# local import (pre-transformers)
from exceptions import ConversionException
from . import transformers as trans #TODO
from .utils import highlight2latex #TODO
from .utils import get_lines #TODO
from .utils import remove_ansi #TODO
from .utils import highlight, ansi2html #TODO

from .transformers.latex import LatexTransformer, rm_math_space

import .utils.strings as strings
import .utils.markdown as markdown_utils
import .utils.datatypefilter.DataTypeFilter as DataTypeFilter

#Try to import the Sphinx exporter.  If the user doesn't have Sphinx isntalled 
#on his/her machine, fail silently.
try:
    from .sphinx_transformer import (SphinxTransformer) #TODO
except ImportError:
    SphinxTransformer = None

#-----------------------------------------------------------------------------
# Globals and constants
#-----------------------------------------------------------------------------

#Standard Jinja2 environment constants
TEMPLATE_PATH = "/../templates/"
TEMPLATE_SKELETON_PATH = "/../templates/skeleton/"
TEMPLATE_EXTENSION = ".tpl"

#Latex Jinja2 constants
LATEX_TEMPLATE_PATH = "/../templates/tex/"
LATEX_TEMPLATE_SKELETON_PATH = "/../templates/tex/skeleton/"
LATEX_TEMPLATE_EXTENSION = ".tplx"

#Special Jinja2 syntax that will not conflict when exporting latex.
LATEX_JINJA_COMMENT_BLOCK = ["((=", "=))"]
LATEX_JINJA_VARIABLE_BLOCK = ["(((", ")))"]
LATEX_JINJA_LOGIC_BLOCK = ["((*", "*))"]

#Jinja2 extensions to load.
JINJA_EXTENSIONS = ['jinja2.ext.loopcontrols']

#Latex substitutions for escaping latex.
LATEX_SUBS = (
    (re.compile(r'\\'), r'\\textbackslash'),
    (re.compile(r'([{}_#%&$])'), r'\\\1'),
    (re.compile(r'~'), r'\~{}'),
    (re.compile(r'\^'), r'\^{}'),
    (re.compile(r'"'), r"''"),
    (re.compile(r'\.\.\.+'), r'\\ldots'),
)

#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------
class Exporter(Configurable):
    """ A Jinja2 base converter templates

    Preprocess the ipynb files, feed it throug jinja templates,
    and spit an converted files and a data object with other data
    should be mostly configurable
    """

    pre_transformer_order = List(['haspyout_transformer'],
        config=True,
        help= """
            An ordered list of pre transformer to apply to the ipynb
            file before running through templates
            """
        )

    tex_environement = Bool(
        False,
        config=True,
        help=" Whether or not the user is exporting to latex.")

    template_file = Unicode(
            '', config=True,
            help="Name of the template file to use")

    fileext = Unicode(
        'txt', config=True, 
        help="Extension of the file that should be written to disk"
        )

    stdout = Bool(
        True, config=True,
        help="""Whether to print the converted ipynb file to stdout
        "use full do diff files without actually writing a new file"""
        )

    write = Bool(
        False, config=True,
        help="""Should the converted notebook file be written to disk
        along with potential extracted resources."""
        )

    #Processors that process the input data prior to the export, set in the 
    #constructor for this class.
    preprocessors = [] 

    def __init__(self, preprocessors={}, jinja_filters={}, config=None, export_format, **kw):
        """ Init a new converter.

        config: the Configurable config object to pass around.

        preprocessors: dict of **availlable** key/value function to run on
                       ipynb json data before conversion to extract/inline file.
                       See `transformer.py` and `ConfigurableTransformers`

                       set the order in which the transformers should apply
                       with the `pre_transformer_order` trait of this class

                       transformers registerd by this key will take precedence on
                       default one.

        jinja_filters: dict of supplementary jinja filter that should be made
                       availlable in template. If those are of Configurable Class type,
                       they will be instanciated with the config object as argument.

                       user defined filter will overwrite the one availlable by default.
        """

        #Merge default config options with user specific override options.
        default_config = self._get_default_options()
        if not config == None:
            default_config._merge(config)
        config = default_config

        #Call the base class constructor
        super(Exporter, self).__init__(config=config, **kw)

        #Create a Latex environment if the user is exporting latex.
        if self.tex_environement:
            self.ext = LATEX_TEMPLATE_EXTENSION
            self.env = Environment(
                loader=FileSystemLoader([
                    os.path.dirname(os.path.realpath(__file__)) + LATEX_TEMPLATE_PATH,
                    os.path.dirname(os.path.realpath(__file__)) + LATEX_TEMPLATE_SKELETON_PATH,
                    ]),
                extensions=JINJA_EXTENSIONS
                )

            #Set special Jinja2 syntax that will not conflict with latex.
            self.env.block_start_string = LATEX_JINJA_LOGIC_BLOCK[0]
            self.env.block_end_string = LATEX_JINJA_LOGIC_BLOCK[1]
            self.env.variable_start_string = LATEX_JINJA_VARIABLE_BLOCK[0]
            self.env.variable_end_string = LATEX_JINJA_VARIABLE_BLOCK[1]
            self.env.comment_start_string = LATEX_JINJA_COMMENT_BLOCK[0]
            self.env.comment_end_string = LATEX_JINJA_COMMENT_BLOCK[1]

        else: #Standard environment
            self.ext = TEMPLATE_EXTENSION
            self.env = Environment(
                loader=FileSystemLoader([
                    os.path.dirname(os.path.realpath(__file__)) + TEMPLATE_PATH,
                    os.path.dirname(os.path.realpath(__file__)) + TEMPLATE_SKELETON_PATH,
                    ]),
                extensions=JINJA_EXTENSIONS
                )

        for name in self.pre_transformer_order:
            # get the user-defined transformer first
            transformer = preprocessors.get(name, getattr(trans, name, None))
            if isinstance(transformer, MetaHasTraits):
                transformer = transformer(config=config)
            self.preprocessors.append(transformer)

        #For compatibility, TODO: remove later.
        self.preprocessors.append(trans.coalesce_streams)
        self.preprocessors.append(trans.ExtractFigureTransformer(config=config))
        self.preprocessors.append(trans.RevealHelpTransformer(config=config))
        self.preprocessors.append(trans.CSSHtmlHeaderTransformer(config=config))
        self.preprocessors.append(LatexTransformer(config=config))

        #Only load the sphinx transformer if the file reference worked 
        #(Sphinx dependencies exist on the user's machine.)
        if SphinxTransformer:
            self.preprocessors.append(SphinxTransformer(config=config))

        #Add filters to the Jinja2 environment
        self.env.filters['filter_data_type'] = DataTypeFilter(config=config)
        self.env.filters['pycomment'] = _python_comment
        self.env.filters['indent'] = indent
        self.env.filters['rm_fake'] = _rm_fake
        self.env.filters['rm_ansi'] = remove_ansi
        self.env.filters['markdown'] = markdown<commit_msg>Moved latex code into latex exporter<commit_after>"""Exporter for the notebook conversion pipeline.

This module defines Exporter, a highly configurable converter
that uses Jinja2 to export notebook files into different format.

You can register both pre-transformers that will act on the notebook format
befor conversion and jinja filter that would then be availlable in the templates
"""

#-----------------------------------------------------------------------------
# Copyright (c) 2013, the IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
from __future__ import print_function, absolute_import

# Stdlib imports
import io
import os
import re

# IPython imports
from IPython.config.configurable import Configurable
from IPython.nbformat import current as nbformat
from IPython.utils.traitlets import MetaHasTraits, Unicode, List, Bool
from IPython.utils.text import indent

# other libs/dependencies
from jinja2 import Environment, FileSystemLoader
from markdown import markdown

# local import (pre-transformers)
from exceptions import ConversionException
from . import transformers as trans #TODO
from .utils import get_lines #TODO
from .utils import remove_ansi #TODO
from .utils import highlight, ansi2html #TODO

import .utils.strings as strings
import .utils.markdown as markdown_utils
import .utils.datatypefilter.DataTypeFilter as DataTypeFilter

#-----------------------------------------------------------------------------
# Globals and constants
#-----------------------------------------------------------------------------

#Standard Jinja2 environment constants
TEMPLATE_PATH = "/../templates/"
TEMPLATE_SKELETON_PATH = "/../templates/skeleton/"
TEMPLATE_EXTENSION = ".tpl"

#Jinja2 extensions to load.
JINJA_EXTENSIONS = ['jinja2.ext.loopcontrols']

#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------
class Exporter(Configurable):
    """ A Jinja2 base converter templates

    Preprocess the ipynb files, feed it throug jinja templates,
    and spit an converted files and a data object with other data
    should be mostly configurable
    """

    pre_transformer_order = List(['haspyout_transformer'],
        config=True,
        help= """
            An ordered list of pre transformer to apply to the ipynb
            file before running through templates
            """
        )

#TODO: Flagged for removal.
    tex_environement = Bool(
        False,
        config=True,
        help=" Whether or not the user is exporting to latex.")

    template_file = Unicode(
            '', config=True,
            help="Name of the template file to use")

    fileext = Unicode(
        'txt', config=True, 
        help="Extension of the file that should be written to disk"
        )

    stdout = Bool(
        True, config=True,
        help="""Whether to print the converted ipynb file to stdout
        "use full do diff files without actually writing a new file"""
        )

    write = Bool(
        False, config=True,
        help="""Should the converted notebook file be written to disk
        along with potential extracted resources."""
        )

    #Processors that process the input data prior to the export, set in the 
    #constructor for this class.
    preprocessors = [] 

    def __init__(self, preprocessors={}, jinja_filters={}, config=None, export_format, **kw):
        """ Init a new converter.

        config: the Configurable config object to pass around.

        preprocessors: dict of **availlable** key/value function to run on
                       ipynb json data before conversion to extract/inline file.
                       See `transformer.py` and `ConfigurableTransformers`

                       set the order in which the transformers should apply
                       with the `pre_transformer_order` trait of this class

                       transformers registerd by this key will take precedence on
                       default one.

        jinja_filters: dict of supplementary jinja filter that should be made
                       availlable in template. If those are of Configurable Class type,
                       they will be instanciated with the config object as argument.

                       user defined filter will overwrite the one availlable by default.
        """

        #Merge default config options with user specific override options.
        default_config = self._get_default_options()
        if not config == None:
            default_config._merge(config)
        config = default_config

        #Call the base class constructor
        super(Exporter, self).__init__(config=config, **kw)

        #Standard environment
        self.ext = TEMPLATE_EXTENSION
        self.env = Environment(
            loader=FileSystemLoader([
                os.path.dirname(os.path.realpath(__file__)) + TEMPLATE_PATH,
                os.path.dirname(os.path.realpath(__file__)) + TEMPLATE_SKELETON_PATH,
                ]),
            extensions=JINJA_EXTENSIONS
            )

        for name in self.pre_transformer_order:
            # get the user-defined transformer first
            transformer = preprocessors.get(name, getattr(trans, name, None))
            if isinstance(transformer, MetaHasTraits):
                transformer = transformer(config=config)
            self.preprocessors.append(transformer)

        #For compatibility, TODO: remove later.
        self.preprocessors.append(trans.coalesce_streams)
        self.preprocessors.append(trans.ExtractFigureTransformer(config=config))
        self.preprocessors.append(trans.RevealHelpTransformer(config=config))
        self.preprocessors.append(trans.CSSHtmlHeaderTransformer(config=config))
        self.preprocessors.append(LatexTransformer(config=config))

        #Only load the sphinx transformer if the file reference worked 
        #(Sphinx dependencies exist on the user's machine.)
        if SphinxTransformer:
            self.preprocessors.append(SphinxTransformer(config=config))

        #Add filters to the Jinja2 environment
        self.env.filters['filter_data_type'] = DataTypeFilter(config=config)
        self.env.filters['pycomment'] = _python_comment
        self.env.filters['indent'] = indent
        self.env.filters['rm_fake'] = _rm_fake
        self.env.filters['rm_ansi'] = remove_ansi
        self.env.filters['markdown'] = markdown
{'fraction_non_alphanumeric': 0.15071816150654324, 'fraction_numerical': 0.0022342802425789976, 'mean_word_length': 2.102792632204397, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4364750', 'n_tokens_mistral': 4099, 'n_tokens_neox': 3884, 'n_words': 1233}