<commit_before>"""Utility for calling pandoc"""
#-----------------------------------------------------------------------------
# Copyright (c) 2013 the IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from __future__ import print_function

# Stdlib imports
import subprocess
import re
import warnings
from io import TextIOWrapper, BytesIO

# IPython imports
from IPython.utils.py3compat import cast_bytes
from IPython.utils.version import check_version
from IPython.utils.process import find_cmd, FindCmdError


from .exceptions import ConversionException


#----------------------------------------------------------------------------
# Preliminary checks.
# Not finding Pandoc is not always fatal so only a warning is issued at the
# module root level so that the import of this module is not fatal. 
#----------------------------------------------------------------------------

# command line to make pandoc print it's version. It is also the
# easiest way to make pandoc return at all.
__pandoc_version_call = ['pandoc', '-v']

class PandocMissing(ConversionException):
    """Exception raised when Pandoc is missing. """
    def __init__(self, cmd, exc, *args, **kwargs):
        super(PandocMissing, self).__init__( "The command '%s' returned an error: %s.\n" %(" ".join(cmd), exc) +
                                             "Please check that pandoc is installed:\n" +
                                             "http://johnmacfarlane.net/pandoc/installing.html" )
        self.exc = exc

    def __bool__(self):
        return False

    __nonzero__ = __bool__
    

def pandoc_available(failmode="return", warn=False):
    """Is pandoc available. Only tries to call Pandoc
    and inform you that it succeeded or failed. 
    
    Parameters
    ----------
     - failmode : string
        either "return" or "raise". If "return" and pandoc
        is not available, will return (False, e) where e is
        the exception returned by subprocess.check_call.
     - warn : bool
        issue a user warning if pandoc is not available.

    Return
    ------
    out : (Bool, Exception)
        On success will return True. On failure and failmode=="return" 
        will return False-valued PandocMissing instance
    """
    
    try:
        find_cmd("pandoc")
        return True
    except FindCmdError as e:
        if warn:
            warnings.warn(
                "Pandoc cannot be found (calling %s failed).\n" % " ".join(alt or __pandoc_version_call) +
                "Please check that pandoc is installed:\n" +
                "http://johnmacfarlane.net/pandoc/installing.html"
            )

        exc = PandocMissing("pandoc", e)
        if failmode == "return":
            return exc
        else:
            raise exc
            

#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------
minimal_version = "1.12.1"

# The following holds cached values about the pandoc executable.
def clean_cache(new=False):
    if new:
        global __cache
        cache = {}
        __cache = cache
    else:
        cache = __cache
        cache.clear()
    
    cache['version_ok'] = False
    cache['version'] = None
    return cache

__cache = clean_cache(new=True)


def pandoc(source, fmt, to, extra_args=None, encoding='utf-8'):
    """Convert an input string in format `from` to format `to` via pandoc.

    Parameters
    ----------
    source : string
      Input string, assumed to be valid format `from`.
    fmt : string
      The name of the input format (markdown, etc.)
    to : string
      The name of the output format (html, etc.)

    Returns
    -------
    out : unicode
      Output as returned by pandoc.

    Exceptions
    ----------
    This function will raise PandocMissing if pandoc is not installed.
    Any error messages generated by pandoc are printed to stderr.

    """
    cmd = ['pandoc', '-f', fmt, '-t', to]
    if extra_args:
        cmd.extend(extra_args)

    # this will raise an exception that will pop us out of here
    check_pandoc_version()
    
    # we can safely continue
    p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    out, _ = p.communicate(cast_bytes(source, encoding))
    out = TextIOWrapper(BytesIO(out), encoding, 'replace').read()
    return out.rstrip('\n')


def get_pandoc_version():
    """Gets the Pandoc version if Pandoc is installed.
    
    Return
    ------
    If the minimal version is not met, it will probe Pandoc for its version, cache it and return that value.
    If the minimal version is met, it will return the cached version and stop probing Pandoc 
    (unless `clean_cache()` is called).

    Exceptions
    ----------
    PandocMissing will be raised if pandoc is unavailable.
    """

    if __cache['version_ok'] and __cache['version']:
        return __cache['version']
    else:
        pandoc_available(failmode="raise")
        out = subprocess.check_output(__pandoc_version_call, universal_newlines=True)
        pv_re = re.compile(r'(\d{0,3}\.\d{0,3}\.\d{0,3})')
        __cache['version'] = version = pv_re.search(out).group(0)
        return version


def check_pandoc_version():
    """Returns True if minimal pandoc version is met.

    Exceptions
    ----------
    PandocMissing will be raised if pandoc is unavailable.
    """
    ok = __cache['version_ok']
    if not ok:
        __cache['version_ok'] = ok = check_version( get_pandoc_version(), minimal_version )
        if not ok:
            warnings.warn( "You are using an old version of pandoc (%s)\n" % __cache['version'] + 
                           "Recommended version is %s.\nTry updating." % minimal_version + 
                           "http://johnmacfarlane.net/pandoc/installing.html.\nContinuing with doubts...",
                           RuntimeWarning, stacklevel=2)
    return __cache['version_ok']

<commit_msg>code shuffling to present the API functions first, and the innards at the end of the file<commit_after>"""Utility for calling pandoc"""
#-----------------------------------------------------------------------------
# Copyright (c) 2013 the IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from __future__ import print_function

# Stdlib imports
import subprocess
import re
import warnings
from io import TextIOWrapper, BytesIO

# IPython imports
from IPython.utils.py3compat import cast_bytes
from IPython.utils.version import check_version
from IPython.utils.process import find_cmd, FindCmdError


from .exceptions import ConversionException

#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------
minimal_version = "1.12.1"

# command line to make pandoc print it's version. It is also the
# easiest way to make pandoc return at all.
__pandoc_version_call = ['pandoc', '-v']


def pandoc(source, fmt, to, extra_args=None, encoding='utf-8'):
    """Convert an input string in format `from` to format `to` via pandoc.

    Parameters
    ----------
    source : string
      Input string, assumed to be valid format `from`.
    fmt : string
      The name of the input format (markdown, etc.)
    to : string
      The name of the output format (html, etc.)

    Returns
    -------
    out : unicode
      Output as returned by pandoc.

    Exceptions
    ----------
    This function will raise PandocMissing if pandoc is not installed.
    Any error messages generated by pandoc are printed to stderr.

    """
    cmd = ['pandoc', '-f', fmt, '-t', to]
    if extra_args:
        cmd.extend(extra_args)

    # this will raise an exception that will pop us out of here
    check_pandoc_version()
    
    # we can safely continue
    p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    out, _ = p.communicate(cast_bytes(source, encoding))
    out = TextIOWrapper(BytesIO(out), encoding, 'replace').read()
    return out.rstrip('\n')


def pandoc_available(failmode="return", warn=False):
    """Is pandoc available. Relies on `IPython.utils.process.find_cmd`.
    
    Parameters
    ----------
     - failmode : string
        either "return" or "raise". See below.
     - warn : bool
        issue a user warning if pandoc is not available.

    Return
    ------
    out : (Bool, Exception)
        On success will return True. On failure and failmode=="return" 
        will return False-valued PandocMissing instance. If failmode is
        anything else, the function will not return but raise PandocMissing.
    """
    
    try:
        find_cmd("pandoc")
        return True
    except FindCmdError as e:
        if warn:
            warnings.warn(
                "Pandoc cannot be found (find_cmd('pandoc') failed).\n"+
                "Please check that pandoc is installed:\n" +
                "http://johnmacfarlane.net/pandoc/installing.html"
            )

        exc = PandocMissing("pandoc", e)
        if failmode == "return":
            return exc
        else:
            raise exc


def get_pandoc_version():
    """Gets the Pandoc version if Pandoc is installed.
    
    Return
    ------
    If the minimal version is not met, it will probe Pandoc for its version, cache it and return that value.
    If the minimal version is met, it will return the cached version and stop probing Pandoc 
    (unless `clean_cache()` is called).

    Exceptions
    ----------
    PandocMissing will be raised if pandoc is unavailable.
    """

    if __cache['version_ok'] and __cache['version']:
        return __cache['version']
    else:
        pandoc_available(failmode="raise")
        out = subprocess.check_output(__pandoc_version_call, universal_newlines=True)
        pv_re = re.compile(r'(\d{0,3}\.\d{0,3}\.\d{0,3})')
        __cache['version'] = version = pv_re.search(out).group(0)
        return version


def check_pandoc_version():
    """Returns True if minimal pandoc version is met.

    Exceptions
    ----------
    PandocMissing will be raised if pandoc is unavailable.
    """
    ok = __cache['version_ok']
    if not ok:
        __cache['version_ok'] = ok = check_version( get_pandoc_version(), minimal_version )
        if not ok:
            warnings.warn( "You are using an old version of pandoc (%s)\n" % __cache['version'] + 
                           "Recommended version is %s.\nTry updating." % minimal_version + 
                           "http://johnmacfarlane.net/pandoc/installing.html.\nContinuing with doubts...",
                           RuntimeWarning, stacklevel=2)
    return __cache['version_ok']

#-----------------------------------------------------------------------------
# Exception handling
#-----------------------------------------------------------------------------
class PandocMissing(ConversionException):
    """Exception raised when Pandoc is missing. """
    def __init__(self, cmd, exc, *args, **kwargs):
        super(PandocMissing, self).__init__( "The command '%s' returned an error: %s.\n" %(" ".join(cmd), exc) +
                                             "Please check that pandoc is installed:\n" +
                                             "http://johnmacfarlane.net/pandoc/installing.html" )
        self.exc = exc

    def __bool__(self):
        return False

    __nonzero__ = __bool__


#-----------------------------------------------------------------------------
# Internal state management
#-----------------------------------------------------------------------------
def clean_cache(new=False):
    if new:
        global __cache
        cache = {}
        __cache = cache
    else:
        cache = __cache
        cache.clear()
    
    cache['version_ok'] = False
    cache['version'] = None
    return cache

# The following holds cache values about the pandoc executable.
__cache = clean_cache(new=True)




{'fraction_non_alphanumeric': 0.20069422530766803, 'fraction_numerical': 0.002840012622278321, 'mean_word_length': 1.6863742318287773, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1535485', 'n_tokens_mistral': 3312, 'n_tokens_neox': 3222, 'n_words': 1097}