<commit_before>
SphinxException = DefineException('SphinxException', 'Sphinx error: {error}', [('error', unicode, False)])

import re
import sys, os, shutil
import subprocess
from copy import deepcopy
from docutils import nodes
from docutils.parsers.rst import Directive, directives
from tempfile import mkdtemp
from sphinx.application import Sphinx
from sphinx.errors import PycodeError, SphinxError

CORE_PATH = os.path.abspath(os.path.split(__file__)[0])

config_overrides = {
    'extensions' : ['sphinx.ext.pngmath','sphinx.ext.graphviz', 'satori.core.models'],
    'templates_path' : [os.path.join(CORE_PATH, 'sphinx_templates')],
    'source_suffix' : '.rst',
    'master_doc' : 'index',
    'project' : u'',
    'copyright': u'',
    'version' : '1',
    'release' : '1',
    'exclude_patterns' : ['_build'],
    'pygments_style' : 'sphinx',
    'pngmath_use_preview' : True,
    'pngmath_latex_preamble' : r'\usepackage{amsmath} \usepackage{polski}',

#    'html_theme' : 'default',
#    'html_theme_path' : [os.path.join(CORE_PATH, 'sphinx_templates')],
#    'html_static_path' : ['_static'],
    'htmlhelp_basename' : '',

    'latex_documents': [('index', 'index.tex', '', '', 'article')],
    'latex_elements': {
            'papersize': 'a4paper',
            'pointsize': '12pt',
            'fontpkg': '',
            'fncychap': '',
            }
}


class TexNode(nodes.Element):
    def __init__(self, *args, **kwargs):
        super(TexNode, self).__init__()
        self.attributes['value'] = kwargs['value']


def visit_tex_node_tex(self, node):
    self.body.append(node.attributes['value'])


def visit_tex_node_other(self, node):
    pass


def depart_tex_node(self, node):
    pass


class PdfInfoDirective(Directive):
    has_content = False
    required_arguments = 0
    optional_arguments = 0
    option_spec = {
            'leftlogo': directives.unchanged,
            'rightlogo': directives.unchanged,
            'date': directives.unchanged,
            'place': directives.unchanged,
            'contest_name': directives.unchanged,
            'contest_date': directives.unchanged,
            }

    def run(self):
        ret = '\n\n'
        if 'leftlogo' in self.options:
            ret += r'\renewcommand{\PDFleftlogo}{\includegraphics[width=2cm]{' + self.options['leftlogo'] + r'}}' + '\n'

        if 'rightlogo' in self.options:
            ret += r'\renewcommand{\PDFrightlogo}{\includegraphics[width=2cm]{' + self.options['rightlogo'] + r'}}' + '\n'

        if 'date' in self.options:
            ret += r'\renewcommand{\PDFdate}{' + self.options['date'] + r'}' + '\n'

        if 'contest_name' in self.options:
            ret += r'\renewcommand{\PDFcontestname}{' + self.options['contest_name'] + r'}' + '\n'

        if 'contest_date' in self.options:
            ret += r'\renewcommand{\PDFcontestdate}{' + self.options['contest_date'] + r'}' + '\n'

        if 'place' in self.options:
            ret += r'\renewcommand{\PDFplace}{' + self.options['place'] + r'}' + '\n'

        return [TexNode(value=ret)]


def setup(app):
    app.add_node(TexNode, latex=(visit_tex_node_tex, depart_tex_node), html=(visit_tex_node_other, depart_tex_node), text=(visit_tex_node_other, depart_tex_node))
    app.add_directive('pdfinfo', PdfInfoDirective)


#TODO: Make this irrelevant
#This is needed because of the bug in sphinx: application.py (the order of extensions loading)
def confToFile(filePath):
    config = open(filePath, 'w')
    for key, val in config_overrides.iteritems():
        if isinstance(val, str) or isinstance(val, (str, unicode)):
            raw = key + ' = "' + val.__str__() + '"\n'
        else:
            raw = key + ' = ' + val.__str__() + '\n'
        config.write(raw)
    config.close()


def render_sphinx(rest, oa_map):
    if not rest:
        to_delete = []
        for name in oa_map:
            oa = oa_map[name]
            if not oa.is_blob:
                continue
            if (name.startswith('_img_') or (name == '_pdf') or (name == '_html')):
                to_delete.append(name)
        for name in to_delete:
            del oa_map[name]
        if 'pdf' in oa_map:
            oa_map['_pdf'] = deepcopy(oa_map['pdf'])
            oa_map['_pdf'].name = '_pdf'
        return oa_map

    srcdir = mkdtemp()
    indexpath = os.path.join(srcdir, 'index.rst')

    rfile = open(indexpath, 'w')
    if isinstance(rest, unicode):
        rfile.write(rest.encode('utf-8'))<commit_msg>Fixed sphinx generation.<commit_after>
SphinxException = DefineException('SphinxException', 'Sphinx error: {error}', [('error', unicode, False)])

import re
import sys, os, shutil
import subprocess
from copy import deepcopy
from docutils import nodes
from docutils.parsers.rst import Directive, directives
from tempfile import mkdtemp
from sphinx.application import Sphinx
from sphinx.errors import PycodeError, SphinxError

CORE_PATH = os.path.abspath(os.path.split(__file__)[0])

config_overrides = {
    'extensions' : ['sphinx.ext.pngmath','sphinx.ext.graphviz', 'satori.core.models'],
    'templates_path' : [os.path.join(CORE_PATH, 'sphinx_templates')],
    'source_suffix' : '.rst',
    'master_doc' : 'index',
    'project' : u'',
    'copyright': u'',
    'version' : '1',
    'release' : '1',
    'exclude_patterns' : ['_build'],
    'pygments_style' : 'sphinx',
    'pngmath_use_preview' : True,
    'pngmath_latex_preamble' : r'\usepackage{amsmath} \usepackage{polski}',

#    'html_theme' : 'default',
#    'html_theme_path' : [os.path.join(CORE_PATH, 'sphinx_templates')],
#    'html_static_path' : ['_static'],
    'htmlhelp_basename' : '',

    'latex_documents': [('index', 'index.tex', '', '', 'article')],
    'latex_elements': {
            'papersize': 'a4paper',
            'pointsize': '12pt',
            'fontpkg': '',
            'fncychap': '',
            }
}


class TexNode(nodes.Element):
    def __init__(self, *args, **kwargs):
        super(TexNode, self).__init__()
        self.attributes['value'] = kwargs['value']


def visit_tex_node_tex(self, node):
    self.body.append(node.attributes['value'])


def visit_tex_node_other(self, node):
    pass


def depart_tex_node(self, node):
    pass


class PdfInfoDirective(Directive):
    has_content = False
    required_arguments = 0
    optional_arguments = 0
    option_spec = {
            'leftlogo': directives.unchanged,
            'rightlogo': directives.unchanged,
            'date': directives.unchanged,
            'place': directives.unchanged,
            'contest_name': directives.unchanged,
            'contest_date': directives.unchanged,
            }

    def run(self):
        ret = '\n\n'
        if 'leftlogo' in self.options:
            ret += r'\renewcommand{\PDFleftlogo}{\includegraphics[width=2cm]{' + self.options['leftlogo'] + r'}}' + '\n'

        if 'rightlogo' in self.options:
            ret += r'\renewcommand{\PDFrightlogo}{\includegraphics[width=2cm]{' + self.options['rightlogo'] + r'}}' + '\n'

        if 'date' in self.options:
            ret += r'\renewcommand{\PDFdate}{' + self.options['date'] + r'}' + '\n'

        if 'contest_name' in self.options:
            ret += r'\renewcommand{\PDFcontestname}{' + self.options['contest_name'] + r'}' + '\n'

        if 'contest_date' in self.options:
            ret += r'\renewcommand{\PDFcontestdate}{' + self.options['contest_date'] + r'}' + '\n'

        if 'place' in self.options:
            ret += r'\renewcommand{\PDFplace}{' + self.options['place'] + r'}' + '\n'

        return [TexNode(value=ret)]


def setup(app):
    app.add_node(TexNode, latex=(visit_tex_node_tex, depart_tex_node), html=(visit_tex_node_other, depart_tex_node), text=(visit_tex_node_other, depart_tex_node))
    app.add_directive('pdfinfo', PdfInfoDirective)


#TODO: Make this irrelevant
#This is needed because of the bug in sphinx: application.py (the order of extensions loading)
def confToFile(filePath):
    config = open(filePath, 'w')
    for key, val in config_overrides.iteritems():
        if isinstance(val, str) or isinstance(val, (str, unicode)):
            raw = key + ' = "' + val.__str__() + '"\n'
        else:
            raw = key + ' = ' + val.__str__() + '\n'
        config.write(raw)
    config.close()


def render_sphinx(rest, oa_map):
    if not rest.strip():
        to_delete = []
        for name in oa_map:
            oa = oa_map[name]
            if not oa.is_blob:
                continue
            if (name.startswith('_img_') or (name == '_pdf') or (name == '_html')):
                to_delete.append(name)
        for name in to_delete:
            del oa_map[name]
        if 'pdf' in oa_map:
            oa_map['_pdf'] = deepcopy(oa_map['pdf'])
            oa_map['_pdf'].name = '_pdf'
        return oa_map

    srcdir = mkdtemp()
    indexpath = os.path.join(srcdir, 'index.rst')

    rfile = open(indexpath, 'w')
    if isinstance(rest, unicode):
        rfile.write(rest.encode('utf-8'))
{'fraction_non_alphanumeric': 0.1550974930362117, 'fraction_numerical': 0.0024512534818941505, 'mean_word_length': 2.887397141619749, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1448144', 'n_tokens_mistral': 3037, 'n_tokens_neox': 2967, 'n_words': 615}