<commit_before>"""
Converts user's reStructuredText input to HTML.
"""
# Standard library imports
import os, pickle, time, shutil, re
from StringIO import StringIO
import logging
logger = logging.getLogger('scipycentral')

# 3rd party (non-Django) imports
from sphinx.application import Sphinx, SphinxError

# Django import
from django.http import HttpResponse
from django.conf import settings

# Internal import
from scipy_central.utils import ensuredir

def compile_rest_to_html(raw_rest):
    """ Compiles the RST string, ``raw_RST`, to HTML.  Performs no
    further checking on the RST string.

    If it is a comment, then we don't modify the HTML with extra class info.
    But we do filter comments to disable hyperlinks.

    Also copy over generated MATH media to the correct directory on the server.
    """
    def sanitize_raw_rest(raw_rest):
        """
        Performs any sanitizing of the user's input.

        Currently performs:
        * Converts string to utf-8 encoding
        * Converts '\\' to '\\\\': i.e. single slash converted to double-slash,
                            because Sphinx converts is back to a single slash
        * Removes hyperlinks to remote images

        TODO(KGD): prevent csv-table directive, raw directive, include directive
        For more, see: http://docutils.sourceforge.net/docs/user/config.html#file-insertion-enabled
        """

        raw_rest = raw_rest.encode('utf-8')

        raw_rest = raw_rest.replace('\\', '\\\\')

        # Replace tabs with 4 spaces: so that source code listings don't get
        # the default 8 spaces that Sphinx/docutils use.
        raw_rest = raw_rest.replace('\t', '    ')

        raw_rest = raw_rest.split('\r\n')

        # Strip  '.. raw::'' directive
        raw = re.compile(r'^(\s*)..(\s*)raw(\s*)::(\s*)')
        for idx, line in enumerate(raw_rest):
            if raw.match(line):
                raw_rest[idx] = ''

        # Remove hyperlinks to remote items: e.g. .. image:: http://badimage.com
        NO_REMOTE = ['image', 'figure', 'literalinclude', 'include', 'csv-table']
        for item in NO_REMOTE:
            remote_re = re.compile(r'^(\s*)..(\s*)' + item + r'(\s*)::(\s*)http')
            for idx, line in enumerate(raw_rest):
                if remote_re.match(line):
                    raw_rest[idx] = '<REMOTE EMBEDDING NOT PERMITTED>'
        return '\r\n'.join(raw_rest)

    def call_sphinx_to_compile(working_dir):
        """
        Changes to the ``working_dir`` directory and compiles the RST files to
        pickle files, according to settings in the conf.py file.

        Returns nothing, but logs if an error occurred.
        """
        build_dir = os.path.abspath(working_dir + os.sep + '_build')
        ensuredir(working_dir)
        ensuredir(build_dir)
        status = StringIO()
        warning = StringIO()
        try:
            app = Sphinx(srcdir=working_dir, confdir=working_dir,
                         outdir = build_dir + os.sep + 'pickle',
                         doctreedir = build_dir + os.sep + 'doctrees',
                         buildername = 'pickle',
                         status = status,
                         warning = warning,
                         freshenv = True,
                         warningiserror = False,
                         tags = [])
<commit_msg>Rather say nothing<commit_after>"""
Converts user's reStructuredText input to HTML.
"""
# Standard library imports
import os, pickle, time, shutil, re
from StringIO import StringIO
import logging
logger = logging.getLogger('scipycentral')

# 3rd party (non-Django) imports
from sphinx.application import Sphinx, SphinxError

# Django import
from django.http import HttpResponse
from django.conf import settings

# Internal import
from scipy_central.utils import ensuredir

def compile_rest_to_html(raw_rest):
    """ Compiles the RST string, ``raw_RST`, to HTML.  Performs no
    further checking on the RST string.

    If it is a comment, then we don't modify the HTML with extra class info.
    But we do filter comments to disable hyperlinks.

    Also copy over generated MATH media to the correct directory on the server.
    """
    def sanitize_raw_rest(raw_rest):
        """
        Performs any sanitizing of the user's input.

        Currently performs:
        * Converts string to utf-8 encoding
        * Converts '\\' to '\\\\': i.e. single slash converted to double-slash,
                            because Sphinx converts is back to a single slash
        * Removes hyperlinks to remote images

        TODO(KGD): prevent csv-table directive, raw directive, include directive
        For more, see: http://docutils.sourceforge.net/docs/user/config.html#file-insertion-enabled
        """

        raw_rest = raw_rest.encode('utf-8')

        raw_rest = raw_rest.replace('\\', '\\\\')

        # Replace tabs with 4 spaces: so that source code listings don't get
        # the default 8 spaces that Sphinx/docutils use.
        raw_rest = raw_rest.replace('\t', '    ')

        raw_rest = raw_rest.split('\r\n')

        # Strip  '.. raw::'' directive
        raw = re.compile(r'^(\s*)..(\s*)raw(\s*)::(\s*)')
        for idx, line in enumerate(raw_rest):
            if raw.match(line):
                raw_rest[idx] = ''

        # Remove hyperlinks to remote items: e.g. .. image:: http://badimage.com
        NO_REMOTE = ['image', 'figure', 'literalinclude', 'include', 'csv-table']
        for item in NO_REMOTE:
            remote_re = re.compile(r'^(\s*)..(\s*)' + item + r'(\s*)::(\s*)http')
            for idx, line in enumerate(raw_rest):
                if remote_re.match(line):
                    raw_rest[idx] = ''
        return '\r\n'.join(raw_rest)

    def call_sphinx_to_compile(working_dir):
        """
        Changes to the ``working_dir`` directory and compiles the RST files to
        pickle files, according to settings in the conf.py file.

        Returns nothing, but logs if an error occurred.
        """
        build_dir = os.path.abspath(working_dir + os.sep + '_build')
        ensuredir(working_dir)
        ensuredir(build_dir)
        status = StringIO()
        warning = StringIO()
        try:
            app = Sphinx(srcdir=working_dir, confdir=working_dir,
                         outdir = build_dir + os.sep + 'pickle',
                         doctreedir = build_dir + os.sep + 'doctrees',
                         buildername = 'pickle',
                         status = status,
                         warning = warning,
                         freshenv = True,
                         warningiserror = False,
                         tags = [])

{'fraction_non_alphanumeric': 0.10699835156601228, 'fraction_numerical': 0.0014985763524651581, 'mean_word_length': 2.089814814814815, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7642611', 'n_tokens_mistral': 1975, 'n_tokens_neox': 1859, 'n_words': 580}