<commit_before>import os
import uuid
import errno
import urllib2
import urllib
import json
import shutil
import subprocess
import tempfile
import re

from django.conf import settings
from judge.math_parser import MathHTMLParser

from markdown_trois import markdown as markdown_trois


refilename = re.compile(r'\\includegraphics{(.*?)}')
PROLOGUE = r'''\documentclass[a4paper]{article}

\usepackage{fullpage}
\usepackage[english]{babel}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{hyperref}
\usepackage{bookmark}
\usepackage[pdftex]{graphicx}
\usepackage{amssymb}
\usepackage{longtable}
\usepackage{tabularx}
\usepackage{tabulary}

\setlength{\parskip}{1em}
\renewcommand{\arraystretch}{1.5}

\usepackage[newcommands]{ragged2e}

\usepackage[utf8]{inputenc}

\usepackage{color}
\usepackage{fancyvrb}

\newenvironment{Highlighting}
    {
        \VerbatimEnvironment
        \begin{Verbatim}[commandchars=\\\{\}]%%
    }
    {\end{Verbatim}}
\newenvironment{Shaded}{}{}
\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{\textbf{{#1}}}}
\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.25,0.31,0.50}{\textbf{{#1}}}}
\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.00,0.56,0.56}{{#1}}}
\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.00,0.56,0.56}{{#1}}}
\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.00,0.56,0.56}{{#1}}}
\newcommand{\CharTok}[1]{\textcolor[rgb]{0.82,0.06,0.25}{{#1}}}
\newcommand{\StringTok}[1]{\textcolor[rgb]{0.82,0.06,0.25}{{#1}}}
\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.56,0.56,0.50}{\textit{{#1}}}}
\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{{#1}}}
\newcommand{\AlertTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}}
\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.56,0.00,0.00}{{#1}}}
\newcommand{\RegionMarkerTok}[1]{{#1}}
\newcommand{\ErrorTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}}
\newcommand{\NormalTok}[1]{{#1}}

\setlength{\parindent}{0pt}

\title{\%s \bf %s}
\author{%s}
\date{\vspace{-5ex}}

\makeatletter
\renewcommand\@seccntformat[1]{\large}
\makeatother

\begin{document}
\maketitle
'''

# \usepackage{listings}

EPILOGUE = r'''
\end{document}'''

LATEX_REPLACE = [
    (u'\u2190', r'\(\leftarrow\)'),
    (u'\u2192', r'\(\rightarrow\)'),
    (u'\u2264', r'\le'),
    (u'\u2265', r'\ge'),
    (u'\u2026', '...'),
    (u'\u2212', '-'),
    ('&le;', r'\le'),
    ('&le;', r'\ge'),
    (r'\lt', '<'),
    (r'\gt', '>'),
]

retable = re.compile(r'(?<=\\begin\{longtable\}\[c\]\{@\{\})l+(?=@\{\}\})')
retablebegin = re.compile(r'\\begin\{longtable\}\[c\]\{@\{\}l+@\{\}\}')


class DollarMath(MathHTMLParser):
    def inline_math(self, math):
        return '$%s$' % math

    def display_math(self, math):
        return '$$%s$$' % math


def format_markdown(markdown):
    return DollarMath.convert(markdown)


def make_latex(markdown, style='problem'):
    html = markdown_trois(markdown, style)
    pandoc = getattr(settings, 'PANDOC_PATH', None)
    if isinstance(html, unicode):
        html = html.encode('utf-8')
    if pandoc is not None:
        proc = subprocess.Popen([pandoc, '-f', 'html', '-t', 'latex'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        return proc.communicate(html)[0]
    else:
        # Sorry, but can't install haskell on openshift.
        stream = urllib2.urlopen('http://johnmacfarlane.net/cgi-bin/trypandoc?%s' % urllib.urlencode({
            'from': 'html', 'to': 'latex', 'text': html
        }))
        result = json.load(stream)
        stream.close()
        return result['result']


def wget_graphics(match):
    path = match.group(1)
    return r'''\immediate\write18{wget %s}
\includegraphics{%s}
''' % (path, path[path.rfind('/') + 1:])


def latex_document(title, author, fragment):
    latex = fragment.replace('\subsection{', '\section{')
    for a, b in LATEX_REPLACE:
        latex = latex.replace(a, b)
    latex = refilename.sub(wget_graphics, latex)
    latex = latex.replace(r'\begin{Highlighting}[]', r'\begin{Highlighting}')
    latex = latex.replace(r'\textbackslash{}(', r'\(')
    latex = latex.replace(r'\textbackslash{})', r'\)')
    latex = latex.replace(r'\textbackslash{}le', r'\le')
    latex = latex.replace(r'\textbackslash{}ge', r'\ge')
    latex = latex.replace(r'\textbackslash{}ne', r'\ne')
    latex = latex.replace(r'\paragraph{', r'\section{')
    latex = retablebegin.sub(lambda m: r'%s\hline' % m.group(0), latex)
    latex = retable.sub(lambda m: '| %s |' % ' | '.join(r'p{0.1\textwidth}' for l in m.group(0)), latex)
    latex = latex.replace(r'\tabularnewline', r'\\ \hline')
    return PROLOGUE % (['Huge', 'LARGE'][len(title) > 30], title.replace('#', r'\#'), author) + latex + EPILOGUE


class LatexPdfMaker(object):
    def __init__(self, source):
        self.dir = os.path.join(getattr(settings, 'PDFLATEX_TEMP_DIR', tempfile.gettempdir()), str(uuid.uuid1()))
        self.proc = None
        self.log = None
        self.source = source
        self.texfile = os.path.join(self.dir, 'output.tex')
        self.pdffile = os.path.join(self.dir, 'output.pdf')

        if isinstance(source, unicode):
            self.source = source.encode('utf-8')

    def make(self):<commit_msg>Give last column most size<commit_after>import os
import uuid
import errno
import urllib2
import urllib
import json
import shutil
import subprocess
import tempfile
import re

from django.conf import settings
from judge.math_parser import MathHTMLParser

from markdown_trois import markdown as markdown_trois


refilename = re.compile(r'\\includegraphics{(.*?)}')
PROLOGUE = r'''\documentclass[a4paper]{article}

\usepackage{fullpage}
\usepackage[english]{babel}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{hyperref}
\usepackage{bookmark}
\usepackage[pdftex]{graphicx}
\usepackage{amssymb}
\usepackage{longtable}
\usepackage{tabularx}
\usepackage{tabulary}

\setlength{\parskip}{1em}
\renewcommand{\arraystretch}{1.5}

\usepackage[newcommands]{ragged2e}

\usepackage[utf8]{inputenc}

\usepackage{color}
\usepackage{fancyvrb}

\newenvironment{Highlighting}
    {
        \VerbatimEnvironment
        \begin{Verbatim}[commandchars=\\\{\}]%%
    }
    {\end{Verbatim}}
\newenvironment{Shaded}{}{}
\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{\textbf{{#1}}}}
\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.25,0.31,0.50}{\textbf{{#1}}}}
\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.00,0.56,0.56}{{#1}}}
\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.00,0.56,0.56}{{#1}}}
\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.00,0.56,0.56}{{#1}}}
\newcommand{\CharTok}[1]{\textcolor[rgb]{0.82,0.06,0.25}{{#1}}}
\newcommand{\StringTok}[1]{\textcolor[rgb]{0.82,0.06,0.25}{{#1}}}
\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.56,0.56,0.50}{\textit{{#1}}}}
\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{{#1}}}
\newcommand{\AlertTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}}
\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.56,0.00,0.00}{{#1}}}
\newcommand{\RegionMarkerTok}[1]{{#1}}
\newcommand{\ErrorTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}}
\newcommand{\NormalTok}[1]{{#1}}

\setlength{\parindent}{0pt}

\title{\%s \bf %s}
\author{%s}
\date{\vspace{-5ex}}

\makeatletter
\renewcommand\@seccntformat[1]{\large}
\makeatother

\begin{document}
\maketitle
'''

# \usepackage{listings}

EPILOGUE = r'''
\end{document}'''

LATEX_REPLACE = [
    (u'\u2190', r'\(\leftarrow\)'),
    (u'\u2192', r'\(\rightarrow\)'),
    (u'\u2264', r'\le'),
    (u'\u2265', r'\ge'),
    (u'\u2026', '...'),
    (u'\u2212', '-'),
    ('&le;', r'\le'),
    ('&le;', r'\ge'),
    (r'\lt', '<'),
    (r'\gt', '>'),
]

retable = re.compile(r'(?<=\\begin\{longtable\}\[c\]\{@\{\})l+(?=@\{\}\})')
retablebegin = re.compile(r'\\begin\{longtable\}\[c\]\{@\{\}l+@\{\}\}')


class DollarMath(MathHTMLParser):
    def inline_math(self, math):
        return '$%s$' % math

    def display_math(self, math):
        return '$$%s$$' % math


def format_markdown(markdown):
    return DollarMath.convert(markdown)


def make_latex(markdown, style='problem'):
    html = markdown_trois(markdown, style)
    pandoc = getattr(settings, 'PANDOC_PATH', None)
    if isinstance(html, unicode):
        html = html.encode('utf-8')
    if pandoc is not None:
        proc = subprocess.Popen([pandoc, '-f', 'html', '-t', 'latex'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        return proc.communicate(html)[0]
    else:
        # Sorry, but can't install haskell on openshift.
        stream = urllib2.urlopen('http://johnmacfarlane.net/cgi-bin/trypandoc?%s' % urllib.urlencode({
            'from': 'html', 'to': 'latex', 'text': html
        }))
        result = json.load(stream)
        stream.close()
        return result['result']


def wget_graphics(match):
    path = match.group(1)
    return r'''\immediate\write18{wget %s}
\includegraphics{%s}
''' % (path, path[path.rfind('/') + 1:])


def latex_document(title, author, fragment):
    latex = fragment.replace('\subsection{', '\section{')
    for a, b in LATEX_REPLACE:
        latex = latex.replace(a, b)
    latex = refilename.sub(wget_graphics, latex)
    latex = latex.replace(r'\begin{Highlighting}[]', r'\begin{Highlighting}')
    latex = latex.replace(r'\textbackslash{}(', r'\(')
    latex = latex.replace(r'\textbackslash{})', r'\)')
    latex = latex.replace(r'\textbackslash{}le', r'\le')
    latex = latex.replace(r'\textbackslash{}ge', r'\ge')
    latex = latex.replace(r'\textbackslash{}ne', r'\ne')
    latex = latex.replace(r'\paragraph{', r'\section{')
    latex = retablebegin.sub(lambda m: r'%s\hline' % m.group(0), latex)
    latex = retable.sub(lambda m: '| %s |' % ' | '.join(['l' for l in m.group(0)[:-1]]+['X']), latex)
    latex = latex.replace(r'\tabularnewline', r'\\ \hline')
    return PROLOGUE % (['Huge', 'LARGE'][len(title) > 30], title.replace('#', r'\#'), author) + latex + EPILOGUE


class LatexPdfMaker(object):
    def __init__(self, source):
        self.dir = os.path.join(getattr(settings, 'PDFLATEX_TEMP_DIR', tempfile.gettempdir()), str(uuid.uuid1()))
        self.proc = None
        self.log = None
        self.source = source
        self.texfile = os.path.join(self.dir, 'output.tex')
        self.pdffile = os.path.join(self.dir, 'output.pdf')

        if isinstance(source, unicode):
            self.source = source.encode('utf-8')

    def make(self):
{'fraction_non_alphanumeric': 0.2224257616660239, 'fraction_numerical': 0.03576937909757038, 'mean_word_length': 4.160696517412935, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4080950', 'n_tokens_mistral': 4109, 'n_tokens_neox': 3866, 'n_words': 583}