<commit_before># -*- coding: utf-8 -*-
# See LICENSE.txt for licensing terms
#$HeadURL$
#$LastChangedDate$
#$LastChangedRevision$

__docformat__ = 'reStructuredText'

import sys
import os
import tempfile
import re
from os.path import abspath, dirname, expanduser, join
from urlparse import urljoin, urlparse, urlunparse
from copy import copy, deepcopy
from optparse import OptionParser
import logging

from docutils.languages import get_language
import docutils.readers.doctree
import docutils.core
import docutils.nodes

import pygments_code_block_directive # install code-block directive

from reportlab.platypus import *
from reportlab.platypus.flowables import _listWrapOn, _Container
#from reportlab.lib.enums import *
#from reportlab.lib.units import *
#from reportlab.lib.pagesizes import *

from flowables import * # our own reportlab flowables

from svgimage import SVGImage
from math_directive import math_node
from math_flowable import Math
from aafigure_directive import Aanode

from log import log

from smartypants import smartyPants

# Is this really the best unescape in the stdlib for '&amp;' => '&'????
from xml.sax.saxutils import unescape, escape

import config

#def escape (x,y):
#    "Dummy escape function to test for excessive escaping"
#    return x
from utils import log, parseRaw
import styles as sty

HAS_PIL = True
try:
    from PIL import Image as PILImage
except ImportError:
    try:
        import Image as PILImage
    except ImportError:
        log.warning("Support for images other than JPG,"
            " is now limited. Please install PIL.")
        HAS_PIL = False

try:
    from PythonMagick import Image as PMImage
    HAS_MAGICK = True
except ImportError:
    HAS_MAGICK = False


try:
    import wordaxe
    from wordaxe.rl.paragraph import Paragraph
    from wordaxe.rl.styles import ParagraphStyle, getSampleStyleSheet
    # PyHnjHyphenator is broken for non-ascii characters, so
    # let's not use it and avoid useless crashes (http://is.gd/19efQ)

    #from wordaxe.PyHnjHyphenator import PyHnjHyphenator
    # If basehyphenator doesn't load, wordaxe is broken
    # pyhyphenator and DCW *may* not load.
    
    from wordaxe.BaseHyphenator import BaseHyphenator
    try:
        from wordaxe.plugins.PyHyphenHyphenator \
            import PyHyphenHyphenator
    except:
        pass
    try:
        from wordaxe.DCWHyphenator import DCWHyphenator
    except:
        pass

except ImportError:
    # log.warning("No support for hyphenation, install wordaxe")
    HAS_WORDAXE = False
else:
    HAS_WORDAXE = True

try:
    import sphinx
    HAS_SPHINX = True
except ImportError:
    HAS_SPHINX = False


class RstToPdf(object):

    def __init__(self, stylesheets=[], language=None,
                 header=None, footer=None,
                 inlinelinks=False, breaklevel=1,
                 font_path=[], style_path=[],
                 fit_mode='shrink', sphinx=False,
                 smarty='0', baseurl=None,
                 repeat_table_rows=False,
                 footnote_backlinks=True, inline_footnotes=False,
                 def_dpi=300, show_frame=False,
                 highlightlang='python', #This one is only used by sphinx
                 basedir=os.getcwd()
                 ):
        global HAS_SPHINX
        self.basedir=basedir
        self.language = language
        self.lowerroman = 'i ii iii iv v vi vii viii ix x xi'.split()
        self.loweralpha = 'abcdefghijklmopqrstuvwxyz'
        self.doc_title = ""
        self.doc_author = ""
        self.decoration = {'header': header,
                           'footer': footer,
                           'endnotes': []}
        # find base path
        if hasattr(sys, 'frozen'):
            PATH = abspath(dirname(sys.executable))
        else:
            PATH = abspath(dirname(__file__))
        stylesheets = [join(PATH, 'styles', 'styles.json'),
                       join(PATH, 'styles', 'default.json')] + stylesheets
        self.styles = sty.StyleSheet(stylesheets,
                                     font_path,
                                     style_path,
                                     def_dpi=def_dpi)
        self.docutils_languages = {}
        self.inlinelinks = inlinelinks
        self.breaklevel = breaklevel
        self.fit_mode = fit_mode
        self.to_unlink = []
        self.smarty = smarty
        self.baseurl = baseurl
        self.repeat_table_rows = repeat_table_rows
        self.footnote_backlinks = footnote_backlinks
        self.inline_footnotes = inline_footnotes
        self.def_dpi = def_dpi
        self.show_frame = show_frame
        self.img_dir = os.path.join(abspath(dirname(__file__)), 'images')

        # Sorry about this, but importing sphinx.roles makes some
        # ordinary documents fail (demo.txt specifically) so
        # I can' t just try to import it outside. I need
        # to do it only if it's requested
        if HAS_SPHINX and sphinx:
            import sphinx.roles
            self.highlightlang = highlightlang
        else:
            HAS_SPHINX = False

        if not self.styles.languages:
            self.styles.languages=[]
            if self.language:
                self.styles.languages.append(self.language)
                self.styles['bodytext'].language = self.language
        
        # Load the docutils language modules for all required languages
        for lang in self.styles.languages:
            try:
                self.docutils_languages[lang] = get_language(lang)
            except ImportError:
                try:
                    self.docutils_languages[lang] = \
                         get_language(lang.split('_', 1)[0])
                except ImportError:
                    log.warning("Can't load Docutils module \
                        for language %s", lang)

        # Load the hyphenators for all required languages
        if HAS_WORDAXE:
            for lang in self.styles.languages:
                if lang.split('_', 1)[0] == 'de':
                    try:
                        wordaxe.hyphRegistry[lang] = DCWHyphenator('de', 5)
                        continue
                    except Exception:
                        # hyphenators may not always be available or crash,
                        # e.g. wordaxe issue 2809074 (http://is.gd/16lqs)
                        log.warning("Can't load wordaxe DCW hyphenator "
                        "for German language, trying Py hyphenator instead")
                    else:
                        continue
                try:
                    wordaxe.hyphRegistry[lang] = PyHyphenHyphenator(lang)
                except Exception:
                    log.warning("Can't load wordaxe Py hyphenator"
                        " for language %s, trying base hyphenator", lang)
                else:
                    continue
                #try:
                    #from wordaxe.PyHnjHyphenator import PyHnjHyphenator
                    #wordaxe.hyphRegistry[lang] = PyHnjHyphenator(
                        #lang, 5, purePython=True)
                #except Exception:
                    #log.warning("Can't load wordaxe PyHnj hyphenator"
                        #" for language %s, trying base hyphenator", lang)
                #else:
                    #continue
                try:
                    wordaxe.hyphRegistry[lang] = BaseHyphenator(lang)
                except Exception:
                    log.warning("Can't even load wordaxe base hyphenator")
            log.info('hyphenation by default in %s , loaded %s',
                self.styles['bodytext'].language,
                ','.join(self.styles.languages))

        self.pending_targets=[]
        self.targets=[]

    def size_for_image_node(self, node):
        imgname = os.path.join(self.basedir,str(node.get("uri")))
        scale = float(node.get('scale', 100))/100

        # Figuring out the size to display of an image is ... annoying.
        # If the user provides a size with a unit, it's simple, adjustUnits
        # will return it in points and we're done.

        # However, often the unit wil be "%" (specially if it's meant for
        # HTML originally. In which case, we will use a percentage of
        # the containing frame.

        # Find the image size in pixels:
        kind = 'direct'
        xdpi, ydpi = self.styles.def_dpi, self.styles.def_dpi
        extension = imgname.split('.')[-1].lower()
        if extension in [
                "ai", "ccx", "cdr", "cgm", "cmx",
                "sk1", "sk", "svg", "xml", "wmf", "fig"]:
            iw, ih = SVGImage(imgname).wrap(0, 0)
            # These are in pt, so convert to px
            iw = iw * xdpi / 72
            ih = ih * ydpi / 72
        elif extension == 'pdf':
            try:
                from pyPdf import pdf
            except:
                log.warning('PDF images are not supported without pypdf')
                return 0, 0, 'direct'
            reader = pdf.PdfFileReader(open(imgname))
            x1, y1, x2, y2 = reader.getPage(0)['/MediaBox']
            # These are in pt, so convert to px
            iw = float((x2-x1) * xdpi / 72)
            ih = float((y2-y1) * ydpi / 72)
        else:
            if HAS_PIL:
                img = PILImage.open(imgname)
                iw, ih = img.size
                xdpi, ydpi = img.info.get('dpi', (xdpi, ydpi))
            elif HAS_MAGICK:
                img = PMImage(imgname)
                iw = img.size().width()
                ih = img.size().height()
                # FIXME: need to figure out how to get the DPI
                # xdpi, ydpi=img.density().???
            else:
                log.warning("Sizing images without PIL "
                            "or PythonMagick, using 100x100")
                iw, ih = 100., 100.

        # Try to get the print resolution from the image itself via PIL.
        # If it fails, assume a DPI of 300, which is pretty much made up,
        # and then a 100% size would be iw*inch/300, so we pass
        # that as the second parameter to adjustUnits
        #
        # Some say the default DPI should be 72. That would mean
        # the largest printable image in A4 paper would be something
        # like 480x640. That would be awful.
        #

        w = node.get('width')
        if w is not None:
            # In this particular case, we want the default unit
            # to be pixels so we work like rst2html
            if w[-1] == '%':
                kind = 'percentage_of_container'
                w=int(w[:-1])
            else:
                # This uses default DPI setting because we
                # are not using the image's "natural size"
                # this is what LaTeX does, according to the
                # docutils mailing list discussion
                w = self.styles.adjustUnits(w, self.styles.tw,
                                            default_unit='px')
        else:
            log.warning("Using image %s without specifying size."
                "Calculating based on image size at %ddpi", imgname, xdpi)
            # No width specified at all, use w in px
            w = iw*inch/xdpi

        h = node.get('height')
        if h is not None and h[-1] != '%':
            h = self.styles.adjustUnits(h, ih*inch/ydpi)
        else:
            # Now, often, only the width is specified!
            # if we don't have a height, we need to keep the
            # aspect ratio, or else it will look ugly
            if h and h[-1]=='%':
                log.error('Setting height as a percentage does **not** work. '\
                          'ignoring height parameter')
            h = w*ih/iw

        # Apply scale factor
        w = w*scale
        h = h*scale

        # And now we have this probably completely bogus size!
        log.info("Image %s size calculated:  %fcm by %fcm",
            imgname, w/cm, h/cm)

        return w, h, kind

    def style_language(self, style):
        """Return language corresponding to this style."""
        try:
            return style.language
        except AttributeError:
            pass
        try:
            return self.styles['bodytext'].language
        except AttributeError:
            # FIXME: this is pretty arbitrary, and will
            # probably not do what you want.
            # however, it should only happen if:
            # * You specified the language of a style
            # * Have no wordaxe installed.
            # Since it only affects hyphenation, and wordaxe is
            # not installed, t should have no effect whatsoever
            return os.environ['LANG'] or 'en'

    def text_for_label(self, label, style):
        """Translate text for label."""
        try:
            text = self.docutils_languages[self.style_language(style)]\
                    .labels[label]
        except KeyError:
            text = label.capitalize()
        return text + ":"

    def text_for_bib_field(self, field, style):
        """Translate text for bibliographic fields."""
        try:
            text = self.docutils_languages[self.style_language(style)]\
                    .bibliographic_fields[field]
        except KeyError:
            text = field
        return text + ":"

    def author_separator(self, style):
        """Return separator string for authors."""
        try:
            sep = self.docutils_languages[self.style_language(style)]\
                    .author_separators[0]
        except KeyError:
            sep = ';'
        return sep + " "

    def styleToFont(self, style):
        '''Takes a style name, returns a font tag for it, like
        "<font face=helvetica size=14 color=red>". Used for inline
        nodes (custom interpreted roles)'''

        try:
            s = self.styles[style]
            bc = s.backColor
            if bc:
                r = '<font face="%s" size="%d" color="#%s" backColor="#%s">'\
                    %(s.fontName, s.fontSize,
                      s.textColor.hexval()[2:], bc.hexval()[2:])
            else:
                r = '<font face="%s" size="%d" color="#%s">' % (
                    s.fontName, s.fontSize, s.textColor.hexval()[2:])
            return r
        except KeyError:
            log.warning('Unknown class %s', style)
            return None

    def gather_pdftext(self, node, depth, replaceEnt=True):
        return ''.join([self.gen_pdftext(n, depth, replaceEnt)
            for n in node.children])

    def gen_pdftext(self, node, depth, replaceEnt=True):
        pre = ""
        post = ""

        log.debug("self.gen_pdftext: %s", node.__class__)
        try:
            log.debug("self.gen_pdftext: %s", node)
        except (UnicodeDecodeError, UnicodeEncodeError):
            log.debug("self.gen_pdftext: %r", node)

        #########################################################
        # SPHINX nodes
        #########################################################
        if HAS_SPHINX and isinstance(node,sphinx.addnodes.desc_signature):
            node.pdftext = self.gather_pdftext(node, depth)
                        
        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.module):
            node.pdftext = self.gather_pdftext(node, depth)
            
        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.desc_addname):
            pre = self.styleToFont("descclassname")
            post = "</font>"
            node.pdftext = pre+self.gather_pdftext(node, depth)+post
            
        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.desc_name):
            pre = self.styleToFont("descname")
            post = "</font>"
            node.pdftext = pre+self.gather_pdftext(node, depth)+post
            
        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.desc_returns):
            pre = self.styleToFont("returns")
            post = "</font>"
            node.pdftext=' &rarr; ' + pre+ self.gather_pdftext(node, depth) + post
            
        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.desc_type):
            pre = self.styleToFont("desctype")
            post = "</font>"
            node.pdftext = pre+self.gather_pdftext(node, depth)+post
            
        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.desc_parameterlist):
            pre=' ('
            post=')'
            t=self.gather_pdftext(node, depth)
            while t and t[0] in ', ':
                t=t[1:]
            node.pdftext = pre+t+post
            
        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.desc_parameter):
            if node.hasattr('noemph'):
                pre = ', '
                post = ''
            else:
                pre = ', <i>'
                post = '</i>'
            pre += self.styleToFont("descparameter")
            post = "</font>"+post
            node.pdftext = pre+self.gather_pdftext(node, depth)+post
            
        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.desc_optional):
            pre =self.styleToFont("optional")+'[</font>, '
            post = self.styleToFont("optional")+']</font>'
            t=self.gather_pdftext(node, depth)
            while t and t[0]in ', ':
                t=t[1:]
            node.pdftext = pre+t+post

        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.desc_annotation):
            node.pdftext = '<i>%s</i>'%self.gather_pdftext(node, depth)

        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.pending_xref):
            node.pdftext = self.gather_pdftext(node, depth)

        #########################################################
        # End of SPHINX nodes
        #########################################################
        
        elif isinstance(node, (docutils.nodes.paragraph,
               docutils.nodes.title, docutils.nodes.subtitle)):
            pre=''
            targets=set(node.get('ids',[])+self.pending_targets)
            self.pending_targets=[]
            for _id in targets:
                if _id not in self.targets:
                    pre+='<a name="%s"/>'%_id
                    self.targets.append(_id)
            node.pdftext = pre+self.gather_pdftext(node, depth) + "\n"

        elif isinstance(node, docutils.nodes.Text):
            node.pdftext = node.astext()
            if replaceEnt:
                node.pdftext = escape(node.pdftext)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, docutils.nodes.strong):
            pre = "<b>"
            post = "</b>"
            node.pdftext = self.gather_pdftext(node, depth)
            #if replaceEnt:
            #    node.pdftext=escape(node.pdftext,True)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, docutils.nodes.emphasis):
            pre = "<i>"
            post = "</i>"
            node.pdftext = self.gather_pdftext(node, depth)
            #if replaceEnt:
            #    node.pdftext=escape(node.pdftext,True)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, docutils.nodes.literal):
            pre = '<font face="%s">' % self.styles['literal'].fontName
            post = "</font>"
            if not self.styles['literal'].hyphenation:
                pre = '<nobr>' + pre
                post += '</nobr>'
            node.pdftext = self.gather_pdftext(node, depth)
            #if replaceEnt:
            #    node.pdftext=escape(node.pdftext,True)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, docutils.nodes.superscript):
            pre = '<super>'
            post = "</super>"
            node.pdftext = self.gather_pdftext(node, depth)
            #if replaceEnt:
                #node.pdftext = escape(node.pdftext, True)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, docutils.nodes.subscript):
            pre = '<sub>'
            post = "</sub>"
            node.pdftext = self.gather_pdftext(node, depth)
            #if replaceEnt:
                #node.pdftext = escape(node.pdftext, True)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, docutils.nodes.title_reference):
            pre = self.styleToFont("title_reference")
            post = "</font>"
            node.pdftext = self.gather_pdftext(node, depth)
            # Fix issue 134
            #if replaceEnt:
                #node.pdftext = escape(node.pdftext, True)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, docutils.nodes.reference):
            uri = node.get('refuri')
            if uri:
                if self.baseurl: # Need to join the uri with the base url
                    uri = urljoin(self.baseurl, uri)
                    
                if urlparse(uri)[0] and self.inlinelinks:
                    # external inline reference
                    post = u' (%s)' % uri
                else:
                    # A plain old link
                    pre += u'<a href="%s" color="%s">' %\
                        (uri, self.styles.linkColor)
                    post = '</a>' + post
            else:
                uri = node.get('refid')
                if uri:
                    pre += u'<a href="#%s" color="%s">' %\
                        (uri, self.styles.linkColor)
                    post = '</a>' + post
            node.pdftext = self.gather_pdftext(node, depth)
            #if replaceEnt:
            #    node.pdftext=escape(node.pdftext,True)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, (docutils.nodes.option_string,
                               docutils.nodes.option_argument)):
            node.pdftext = node.astext()
            if replaceEnt:
                node.pdftext = escape(node.pdftext)

        elif isinstance(node, (docutils.nodes.header, docutils.nodes.footer)):
            node.pdftext = self.gather_pdftext(node, depth)
            if replaceEnt:
                node.pdftext = escape(node.pdftext)

            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, (docutils.nodes.system_message,
                               docutils.nodes.problematic)):
            pre = '<font color="red">'
            post = "</font>"
            node.pdftext = self.gather_pdftext(node, depth)
            if replaceEnt:
                node.pdftext = escape(node.pdftext)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, docutils.nodes.generated):
            node.pdftext = self.gather_pdftext(node, depth)
            if replaceEnt:
                node.pdftext = escape(node.pdftext)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, docutils.nodes.image):
            # First see if the image file exists, or else,
            # use image-missing.png
            uri=node.get('uri')
            if not os.path.exists(uri):
                log.error("Missing image file: %s"%uri)
                uri=os.path.join(self.img_dir, 'image-missing.png')
                w, h = 1*cm, 1*cm
            else:
                w, h, kind = self.size_for_image_node(node)
            alignment=node.get('align', 'CENTER').lower()
            if alignment in ('top', 'middle', 'bottom'):
                align='valign="%s"'%alignment
            else:
                align=''

            node.pdftext = '<img src="%s" width="%f" height="%f" %s/>'%\
                (uri, w, h, align)

        elif isinstance(node, math_node):
            mf = Math(node.math_data)
            w, h = mf.wrap(0, 0)
            descent = mf.descent()
            img = mf.genImage()
            self.to_unlink.append(img)
            node.pdftext = '<img src="%s" width=%f height=%f valign=%f/>' % (
                img, w, h, -descent)

        elif isinstance(node, docutils.nodes.footnote_reference):
            # TODO: when used in Sphinx, all footnotes are autonumbered
            anchors=''
            for i in node['ids']:
                if i not in self.targets:
                    anchors+='<a name="%s"/>' % i
                    self.targets.append(i)
            node.pdftext = u'%s<super><a href="%s" color="%s">%s</a></super>'%\
                (anchors, '#' + node.astext(),
                 self.styles.linkColor, node.astext())
                 
        elif isinstance(node, docutils.nodes.citation_reference):
            anchors=''
            for i in node['ids']:
                if i not in self.targets:
                    anchors +='<a name="%s"/>' % i
                    self.targets.append(i)
            node.pdftext = u'%s[<a href="%s" color="%s">%s</a>]'%\
                (anchors, '#' + node.astext(),
                 self.styles.linkColor, node.astext())

        elif isinstance(node, docutils.nodes.target):
            if node['ids'][0] not in self.targets:
                pre = u'<a name="%s"/>' % node['ids'][0]
                self.targets.append(node['ids'][0])
            node.pdftext = self.gather_pdftext(node, depth)
            if replaceEnt:
                node.pdftext = escape(node.pdftext)
            node.pdftext = pre + node.pdftext

        elif isinstance(node, docutils.nodes.inline):
            ftag = self.styleToFont(node['classes'][0])
            if ftag:
                node.pdftext = "%s%s</font>"%\
                    (ftag, self.gather_pdftext(node, depth))
            else:
                node.pdftext = self.gather_pdftext(node, depth)
                
        elif isinstance(node,docutils.nodes.literal_block):
            node.pdftext = self.gather_pdftext(node, depth)

            
        else:
            # With sphinx you will get hundreds of these
            #if not HAS_SPHINX:
            log.warning("Unkn. node (self.gen_pdftext): %s",
                    node.__class__)
            try:
                log.debug(node)
            except (UnicodeDecodeError, UnicodeEncodeError):
                log.debug(repr(node))
            node.pdftext = self.gather_pdftext(node, depth)

        try:
            log.debug("self.gen_pdftext: %s" % node.pdftext)
        except UnicodeDecodeError:
            pass
        # Try to be clever about when to use smartypants
        if node.__class__ in (docutils.nodes.paragraph,
                docutils.nodes.block_quote, docutils.nodes.title):
            node.pdftext = smartyPants(node.pdftext, self.smarty)

        return node.pdftext

    def gen_elements(self, node, depth, style=None):

        log.debug("gen_elements: %s", node.__class__)
        try:
            log.debug("gen_elements: %s", node)
        except (UnicodeDecodeError, UnicodeEncodeError):
            log.debug("gen_elements: %r", node)

        try:
            if node['classes'] and node['classes'][0]:
                # FIXME: Supports only one class, sorry ;-)
                if self.styles.StyleSheet.has_key(node['classes'][0]):
                    style = self.styles[node['classes'][0]]
                else:
                    log.info("Unknown class %s, ignoring.",
                        node['classes'][0])
        except TypeError: # Happens when a docutils.node.Text reaches here
            pass

        if style is None or style == self.styles['bodytext']:
            style = self.styles.styleForNode(node)

        if isinstance(node, docutils.nodes.document):
            node.elements = self.gather_elements(node, depth, style=style)

        elif HAS_SPHINX and isinstance(node, (sphinx.addnodes.glossary,
                                              sphinx.addnodes.start_of_file)):
            node.elements = self.gather_elements(node, depth, style=style)
            
        elif HAS_SPHINX and isinstance(node, (sphinx.addnodes.index)):
            try:
                self.pending_targets.append(node['entries'][0][2])
            except IndexError:
                if node['entries']:
                    log.error("Can't process index entry: %s",node['entries'])
            node.elements = []

        elif isinstance(node, math_node):
            node.elements = [Math(node.math_data)]

        #######################
        ## Tables
        #######################

        elif isinstance(node, docutils.nodes.table):
            node.elements = [Spacer(0, self.styles['table'].spaceBefore)] + \
                            self.gather_elements(node, depth, style=style) +\
                            [Spacer(0, self.styles['table'].spaceAfter)]

        elif isinstance(node, docutils.nodes.tgroup):
            rows = []
            colwidths = []
            hasHead = False
            headRows = 0
            for n in node.children:
                if isinstance(n, docutils.nodes.thead):
                    hasHead = True
                    for row in n.children:
                        r = []
                        for cell in row.children:
                            r.append(cell)
                        rows.append(r)
                    headRows = len(rows)
                elif isinstance(n, docutils.nodes.tbody):
                    for row in n.children:
                        r = []
                        for cell in row.children:
                            r.append(cell)
                        rows.append(r)
                elif isinstance(n, docutils.nodes.colspec):
                    colwidths.append(int(n['colwidth']))

            # colWidths are in no specific unit, really. Maybe ems.
            # Convert them to %
            colwidths=map(int, colwidths)
            tot=sum(colwidths)
            colwidths=["%s%%"%((100.*w)/tot) for w in colwidths]

            if 'colWidths' in style.__dict__:
                colwidths[:len(style.colWidths)]=style.colWidths

            spans = self.filltable(rows)

            data = []
            cellStyles = []
            rowids = range(0, len(rows))
            for row, i in zip(rows, rowids):
                r = []
                j = 0
                for cell in row:
                    if isinstance(cell, str):
                        r.append("")
                    else:
                        # I honestly have no idea what the next line does
                        # (<NAME>, May 25th, 2009)
                        ell = self.gather_elements(cell, depth, style=
                            i < headRows and self.styles['table-heading'] \
                            or style)
                        #if len(ell) == 1:
                            # Experiment: if the cell has a single element,
                            # extract its  class and use it for the cell.
                            # That way, you can have cells with specific
                            # background colors, at least.
                            #
                            # Experiment killed ;-)
                            # You can do that and more using table styles now!
                            #try:
                                #cellStyles += \
                                    #self.styles.pStyleToTStyle(ell[0].style,
                                                               #j, i)
                            ## Fix for issue 85: only do it if it has a style.
                            #except AttributeError:
                                #pass
                        r.append(ell)
                    j += 1
                data.append(r)

            st = TableStyle(spans)
            if 'commands' in self.styles['table'].__dict__:
                for cmd in self.styles['table'].commands:
                    st.add(*cmd)
            if 'commands' in style.__dict__:
                for cmd in style.commands:
                    st.add(*cmd)
            for cmd in cellStyles:
                st.add(*cmd)

            if hasHead:
                for cmd in self.styles.tstyleHead(headRows):
                    st.add(*cmd)
            rtr = self.repeat_table_rows

            node.elements = [DelayedTable(data, colwidths, st, rtr)]

        elif isinstance(node, docutils.nodes.title):
            # Special cases: (Not sure this is right ;-)
            if isinstance(node.parent, docutils.nodes.document):
                node.elements = [Paragraph(self.gen_pdftext(node, depth),
                                           self.styles['title'])]
                self.doc_title = unicode(self.gen_pdftext(node, depth)).strip()
            elif isinstance(node.parent, docutils.nodes.topic):
                node.elements = [Paragraph(self.gen_pdftext(node, depth),
                                           self.styles['topic-title'])]
            elif isinstance(node.parent, docutils.nodes.Admonition):
                node.elements = [Paragraph(self.gen_pdftext(node, depth),
                                           self.styles['admonition-title'])]
            elif isinstance(node.parent, docutils.nodes.table):
                node.elements = [Paragraph(self.gen_pdftext(node, depth),
                                           self.styles['table-title'])]
            elif isinstance(node.parent, docutils.nodes.sidebar):
                node.elements = [Paragraph(self.gen_pdftext(node, depth),
                                           self.styles['sidebar-title'])]
            else:
                # Section/Subsection/etc.
                text = self.gen_pdftext(node, depth)
                fch = node.children[0]
                if isinstance(fch, docutils.nodes.generated) and \
                    fch['classes'] == ['sectnum']:
                    snum = fch.astext()
                else:
                    snum = None
                key = node.get('refid')
                if reportlab.Version > '2.1':
                    node.elements = [ Heading(text, 
                            self.styles['heading%d'%min(depth, 6)],
                            level=depth-1,
                            label=key,
                            parent_id=(node.parent.get('ids', [None]) or [None])[0]
                            )]
                else:
                    node.elements = [Paragraph(text,
                            self.styles['heading%d'%min(depth, 4)]), elem]
                if depth <= self.breaklevel:
                    node.elements.insert(0, MyPageBreak())

        elif isinstance(node, docutils.nodes.subtitle):
            if isinstance(node.parent, docutils.nodes.sidebar):
                node.elements = [Paragraph(self.gen_pdftext(node, depth),
                    self.styles['sidebar-subtitle'])]
            elif isinstance(node.parent, docutils.nodes.document):
                node.elements = [Paragraph(self.gen_pdftext(node, depth),
                    self.styles['subtitle'])]

        elif HAS_SPHINX and isinstance(node,
                sphinx.addnodes.compact_paragraph):
            node.elements = self.gather_elements(node, depth, style=style)
            
        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.module):
            node.elements = [Reference('module-'+node['modname'])]

        elif isinstance(node, docutils.nodes.paragraph):
            node.elements = [Paragraph(self.gen_pdftext(node, depth), style)]

        elif isinstance(node, docutils.nodes.docinfo):<commit_msg>Keep it compatible with RL 2.1 (untested)<commit_after># -*- coding: utf-8 -*-
# See LICENSE.txt for licensing terms
#$HeadURL$
#$LastChangedDate$
#$LastChangedRevision$

__docformat__ = 'reStructuredText'

import sys
import os
import tempfile
import re
from os.path import abspath, dirname, expanduser, join
from urlparse import urljoin, urlparse, urlunparse
from copy import copy, deepcopy
from optparse import OptionParser
import logging

from docutils.languages import get_language
import docutils.readers.doctree
import docutils.core
import docutils.nodes

import pygments_code_block_directive # install code-block directive

from reportlab.platypus import *
from reportlab.platypus.flowables import _listWrapOn, _Container
#from reportlab.lib.enums import *
#from reportlab.lib.units import *
#from reportlab.lib.pagesizes import *

from flowables import * # our own reportlab flowables

from svgimage import SVGImage
from math_directive import math_node
from math_flowable import Math
from aafigure_directive import Aanode

from log import log

from smartypants import smartyPants

# Is this really the best unescape in the stdlib for '&amp;' => '&'????
from xml.sax.saxutils import unescape, escape

import config

#def escape (x,y):
#    "Dummy escape function to test for excessive escaping"
#    return x
from utils import log, parseRaw
import styles as sty

HAS_PIL = True
try:
    from PIL import Image as PILImage
except ImportError:
    try:
        import Image as PILImage
    except ImportError:
        log.warning("Support for images other than JPG,"
            " is now limited. Please install PIL.")
        HAS_PIL = False

try:
    from PythonMagick import Image as PMImage
    HAS_MAGICK = True
except ImportError:
    HAS_MAGICK = False


try:
    import wordaxe
    from wordaxe.rl.paragraph import Paragraph
    from wordaxe.rl.styles import ParagraphStyle, getSampleStyleSheet
    # PyHnjHyphenator is broken for non-ascii characters, so
    # let's not use it and avoid useless crashes (http://is.gd/19efQ)

    #from wordaxe.PyHnjHyphenator import PyHnjHyphenator
    # If basehyphenator doesn't load, wordaxe is broken
    # pyhyphenator and DCW *may* not load.
    
    from wordaxe.BaseHyphenator import BaseHyphenator
    try:
        from wordaxe.plugins.PyHyphenHyphenator \
            import PyHyphenHyphenator
    except:
        pass
    try:
        from wordaxe.DCWHyphenator import DCWHyphenator
    except:
        pass

except ImportError:
    # log.warning("No support for hyphenation, install wordaxe")
    HAS_WORDAXE = False
else:
    HAS_WORDAXE = True

try:
    import sphinx
    HAS_SPHINX = True
except ImportError:
    HAS_SPHINX = False


class RstToPdf(object):

    def __init__(self, stylesheets=[], language=None,
                 header=None, footer=None,
                 inlinelinks=False, breaklevel=1,
                 font_path=[], style_path=[],
                 fit_mode='shrink', sphinx=False,
                 smarty='0', baseurl=None,
                 repeat_table_rows=False,
                 footnote_backlinks=True, inline_footnotes=False,
                 def_dpi=300, show_frame=False,
                 highlightlang='python', #This one is only used by sphinx
                 basedir=os.getcwd()
                 ):
        global HAS_SPHINX
        self.basedir=basedir
        self.language = language
        self.lowerroman = 'i ii iii iv v vi vii viii ix x xi'.split()
        self.loweralpha = 'abcdefghijklmopqrstuvwxyz'
        self.doc_title = ""
        self.doc_author = ""
        self.decoration = {'header': header,
                           'footer': footer,
                           'endnotes': []}
        # find base path
        if hasattr(sys, 'frozen'):
            PATH = abspath(dirname(sys.executable))
        else:
            PATH = abspath(dirname(__file__))
        stylesheets = [join(PATH, 'styles', 'styles.json'),
                       join(PATH, 'styles', 'default.json')] + stylesheets
        self.styles = sty.StyleSheet(stylesheets,
                                     font_path,
                                     style_path,
                                     def_dpi=def_dpi)
        self.docutils_languages = {}
        self.inlinelinks = inlinelinks
        self.breaklevel = breaklevel
        self.fit_mode = fit_mode
        self.to_unlink = []
        self.smarty = smarty
        self.baseurl = baseurl
        self.repeat_table_rows = repeat_table_rows
        self.footnote_backlinks = footnote_backlinks
        self.inline_footnotes = inline_footnotes
        self.def_dpi = def_dpi
        self.show_frame = show_frame
        self.img_dir = os.path.join(abspath(dirname(__file__)), 'images')

        # Sorry about this, but importing sphinx.roles makes some
        # ordinary documents fail (demo.txt specifically) so
        # I can' t just try to import it outside. I need
        # to do it only if it's requested
        if HAS_SPHINX and sphinx:
            import sphinx.roles
            self.highlightlang = highlightlang
        else:
            HAS_SPHINX = False

        if not self.styles.languages:
            self.styles.languages=[]
            if self.language:
                self.styles.languages.append(self.language)
                self.styles['bodytext'].language = self.language
        
        # Load the docutils language modules for all required languages
        for lang in self.styles.languages:
            try:
                self.docutils_languages[lang] = get_language(lang)
            except ImportError:
                try:
                    self.docutils_languages[lang] = \
                         get_language(lang.split('_', 1)[0])
                except ImportError:
                    log.warning("Can't load Docutils module \
                        for language %s", lang)

        # Load the hyphenators for all required languages
        if HAS_WORDAXE:
            for lang in self.styles.languages:
                if lang.split('_', 1)[0] == 'de':
                    try:
                        wordaxe.hyphRegistry[lang] = DCWHyphenator('de', 5)
                        continue
                    except Exception:
                        # hyphenators may not always be available or crash,
                        # e.g. wordaxe issue 2809074 (http://is.gd/16lqs)
                        log.warning("Can't load wordaxe DCW hyphenator "
                        "for German language, trying Py hyphenator instead")
                    else:
                        continue
                try:
                    wordaxe.hyphRegistry[lang] = PyHyphenHyphenator(lang)
                except Exception:
                    log.warning("Can't load wordaxe Py hyphenator"
                        " for language %s, trying base hyphenator", lang)
                else:
                    continue
                #try:
                    #from wordaxe.PyHnjHyphenator import PyHnjHyphenator
                    #wordaxe.hyphRegistry[lang] = PyHnjHyphenator(
                        #lang, 5, purePython=True)
                #except Exception:
                    #log.warning("Can't load wordaxe PyHnj hyphenator"
                        #" for language %s, trying base hyphenator", lang)
                #else:
                    #continue
                try:
                    wordaxe.hyphRegistry[lang] = BaseHyphenator(lang)
                except Exception:
                    log.warning("Can't even load wordaxe base hyphenator")
            log.info('hyphenation by default in %s , loaded %s',
                self.styles['bodytext'].language,
                ','.join(self.styles.languages))

        self.pending_targets=[]
        self.targets=[]

    def size_for_image_node(self, node):
        imgname = os.path.join(self.basedir,str(node.get("uri")))
        scale = float(node.get('scale', 100))/100

        # Figuring out the size to display of an image is ... annoying.
        # If the user provides a size with a unit, it's simple, adjustUnits
        # will return it in points and we're done.

        # However, often the unit wil be "%" (specially if it's meant for
        # HTML originally. In which case, we will use a percentage of
        # the containing frame.

        # Find the image size in pixels:
        kind = 'direct'
        xdpi, ydpi = self.styles.def_dpi, self.styles.def_dpi
        extension = imgname.split('.')[-1].lower()
        if extension in [
                "ai", "ccx", "cdr", "cgm", "cmx",
                "sk1", "sk", "svg", "xml", "wmf", "fig"]:
            iw, ih = SVGImage(imgname).wrap(0, 0)
            # These are in pt, so convert to px
            iw = iw * xdpi / 72
            ih = ih * ydpi / 72
        elif extension == 'pdf':
            try:
                from pyPdf import pdf
            except:
                log.warning('PDF images are not supported without pypdf')
                return 0, 0, 'direct'
            reader = pdf.PdfFileReader(open(imgname))
            x1, y1, x2, y2 = reader.getPage(0)['/MediaBox']
            # These are in pt, so convert to px
            iw = float((x2-x1) * xdpi / 72)
            ih = float((y2-y1) * ydpi / 72)
        else:
            if HAS_PIL:
                img = PILImage.open(imgname)
                iw, ih = img.size
                xdpi, ydpi = img.info.get('dpi', (xdpi, ydpi))
            elif HAS_MAGICK:
                img = PMImage(imgname)
                iw = img.size().width()
                ih = img.size().height()
                # FIXME: need to figure out how to get the DPI
                # xdpi, ydpi=img.density().???
            else:
                log.warning("Sizing images without PIL "
                            "or PythonMagick, using 100x100")
                iw, ih = 100., 100.

        # Try to get the print resolution from the image itself via PIL.
        # If it fails, assume a DPI of 300, which is pretty much made up,
        # and then a 100% size would be iw*inch/300, so we pass
        # that as the second parameter to adjustUnits
        #
        # Some say the default DPI should be 72. That would mean
        # the largest printable image in A4 paper would be something
        # like 480x640. That would be awful.
        #

        w = node.get('width')
        if w is not None:
            # In this particular case, we want the default unit
            # to be pixels so we work like rst2html
            if w[-1] == '%':
                kind = 'percentage_of_container'
                w=int(w[:-1])
            else:
                # This uses default DPI setting because we
                # are not using the image's "natural size"
                # this is what LaTeX does, according to the
                # docutils mailing list discussion
                w = self.styles.adjustUnits(w, self.styles.tw,
                                            default_unit='px')
        else:
            log.warning("Using image %s without specifying size."
                "Calculating based on image size at %ddpi", imgname, xdpi)
            # No width specified at all, use w in px
            w = iw*inch/xdpi

        h = node.get('height')
        if h is not None and h[-1] != '%':
            h = self.styles.adjustUnits(h, ih*inch/ydpi)
        else:
            # Now, often, only the width is specified!
            # if we don't have a height, we need to keep the
            # aspect ratio, or else it will look ugly
            if h and h[-1]=='%':
                log.error('Setting height as a percentage does **not** work. '\
                          'ignoring height parameter')
            h = w*ih/iw

        # Apply scale factor
        w = w*scale
        h = h*scale

        # And now we have this probably completely bogus size!
        log.info("Image %s size calculated:  %fcm by %fcm",
            imgname, w/cm, h/cm)

        return w, h, kind

    def style_language(self, style):
        """Return language corresponding to this style."""
        try:
            return style.language
        except AttributeError:
            pass
        try:
            return self.styles['bodytext'].language
        except AttributeError:
            # FIXME: this is pretty arbitrary, and will
            # probably not do what you want.
            # however, it should only happen if:
            # * You specified the language of a style
            # * Have no wordaxe installed.
            # Since it only affects hyphenation, and wordaxe is
            # not installed, t should have no effect whatsoever
            return os.environ['LANG'] or 'en'

    def text_for_label(self, label, style):
        """Translate text for label."""
        try:
            text = self.docutils_languages[self.style_language(style)]\
                    .labels[label]
        except KeyError:
            text = label.capitalize()
        return text + ":"

    def text_for_bib_field(self, field, style):
        """Translate text for bibliographic fields."""
        try:
            text = self.docutils_languages[self.style_language(style)]\
                    .bibliographic_fields[field]
        except KeyError:
            text = field
        return text + ":"

    def author_separator(self, style):
        """Return separator string for authors."""
        try:
            sep = self.docutils_languages[self.style_language(style)]\
                    .author_separators[0]
        except KeyError:
            sep = ';'
        return sep + " "

    def styleToFont(self, style):
        '''Takes a style name, returns a font tag for it, like
        "<font face=helvetica size=14 color=red>". Used for inline
        nodes (custom interpreted roles)'''

        try:
            s = self.styles[style]
            bc = s.backColor
            if bc:
                r = '<font face="%s" size="%d" color="#%s" backColor="#%s">'\
                    %(s.fontName, s.fontSize,
                      s.textColor.hexval()[2:], bc.hexval()[2:])
            else:
                r = '<font face="%s" size="%d" color="#%s">' % (
                    s.fontName, s.fontSize, s.textColor.hexval()[2:])
            return r
        except KeyError:
            log.warning('Unknown class %s', style)
            return None

    def gather_pdftext(self, node, depth, replaceEnt=True):
        return ''.join([self.gen_pdftext(n, depth, replaceEnt)
            for n in node.children])

    def gen_pdftext(self, node, depth, replaceEnt=True):
        pre = ""
        post = ""

        log.debug("self.gen_pdftext: %s", node.__class__)
        try:
            log.debug("self.gen_pdftext: %s", node)
        except (UnicodeDecodeError, UnicodeEncodeError):
            log.debug("self.gen_pdftext: %r", node)

        #########################################################
        # SPHINX nodes
        #########################################################
        if HAS_SPHINX and isinstance(node,sphinx.addnodes.desc_signature):
            node.pdftext = self.gather_pdftext(node, depth)
                        
        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.module):
            node.pdftext = self.gather_pdftext(node, depth)
            
        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.desc_addname):
            pre = self.styleToFont("descclassname")
            post = "</font>"
            node.pdftext = pre+self.gather_pdftext(node, depth)+post
            
        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.desc_name):
            pre = self.styleToFont("descname")
            post = "</font>"
            node.pdftext = pre+self.gather_pdftext(node, depth)+post
            
        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.desc_returns):
            pre = self.styleToFont("returns")
            post = "</font>"
            node.pdftext=' &rarr; ' + pre+ self.gather_pdftext(node, depth) + post
            
        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.desc_type):
            pre = self.styleToFont("desctype")
            post = "</font>"
            node.pdftext = pre+self.gather_pdftext(node, depth)+post
            
        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.desc_parameterlist):
            pre=' ('
            post=')'
            t=self.gather_pdftext(node, depth)
            while t and t[0] in ', ':
                t=t[1:]
            node.pdftext = pre+t+post
            
        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.desc_parameter):
            if node.hasattr('noemph'):
                pre = ', '
                post = ''
            else:
                pre = ', <i>'
                post = '</i>'
            pre += self.styleToFont("descparameter")
            post = "</font>"+post
            node.pdftext = pre+self.gather_pdftext(node, depth)+post
            
        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.desc_optional):
            pre =self.styleToFont("optional")+'[</font>, '
            post = self.styleToFont("optional")+']</font>'
            t=self.gather_pdftext(node, depth)
            while t and t[0]in ', ':
                t=t[1:]
            node.pdftext = pre+t+post

        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.desc_annotation):
            node.pdftext = '<i>%s</i>'%self.gather_pdftext(node, depth)

        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.pending_xref):
            node.pdftext = self.gather_pdftext(node, depth)

        #########################################################
        # End of SPHINX nodes
        #########################################################
        
        elif isinstance(node, (docutils.nodes.paragraph,
               docutils.nodes.title, docutils.nodes.subtitle)):
            pre=''
            targets=set(node.get('ids',[])+self.pending_targets)
            self.pending_targets=[]
            for _id in targets:
                if _id not in self.targets:
                    pre+='<a name="%s"/>'%_id
                    self.targets.append(_id)
            node.pdftext = pre+self.gather_pdftext(node, depth) + "\n"

        elif isinstance(node, docutils.nodes.Text):
            node.pdftext = node.astext()
            if replaceEnt:
                node.pdftext = escape(node.pdftext)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, docutils.nodes.strong):
            pre = "<b>"
            post = "</b>"
            node.pdftext = self.gather_pdftext(node, depth)
            #if replaceEnt:
            #    node.pdftext=escape(node.pdftext,True)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, docutils.nodes.emphasis):
            pre = "<i>"
            post = "</i>"
            node.pdftext = self.gather_pdftext(node, depth)
            #if replaceEnt:
            #    node.pdftext=escape(node.pdftext,True)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, docutils.nodes.literal):
            pre = '<font face="%s">' % self.styles['literal'].fontName
            post = "</font>"
            if not self.styles['literal'].hyphenation:
                pre = '<nobr>' + pre
                post += '</nobr>'
            node.pdftext = self.gather_pdftext(node, depth)
            #if replaceEnt:
            #    node.pdftext=escape(node.pdftext,True)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, docutils.nodes.superscript):
            pre = '<super>'
            post = "</super>"
            node.pdftext = self.gather_pdftext(node, depth)
            #if replaceEnt:
                #node.pdftext = escape(node.pdftext, True)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, docutils.nodes.subscript):
            pre = '<sub>'
            post = "</sub>"
            node.pdftext = self.gather_pdftext(node, depth)
            #if replaceEnt:
                #node.pdftext = escape(node.pdftext, True)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, docutils.nodes.title_reference):
            pre = self.styleToFont("title_reference")
            post = "</font>"
            node.pdftext = self.gather_pdftext(node, depth)
            # Fix issue 134
            #if replaceEnt:
                #node.pdftext = escape(node.pdftext, True)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, docutils.nodes.reference):
            uri = node.get('refuri')
            if uri:
                if self.baseurl: # Need to join the uri with the base url
                    uri = urljoin(self.baseurl, uri)
                    
                if urlparse(uri)[0] and self.inlinelinks:
                    # external inline reference
                    post = u' (%s)' % uri
                else:
                    # A plain old link
                    pre += u'<a href="%s" color="%s">' %\
                        (uri, self.styles.linkColor)
                    post = '</a>' + post
            else:
                uri = node.get('refid')
                if uri:
                    pre += u'<a href="#%s" color="%s">' %\
                        (uri, self.styles.linkColor)
                    post = '</a>' + post
            node.pdftext = self.gather_pdftext(node, depth)
            #if replaceEnt:
            #    node.pdftext=escape(node.pdftext,True)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, (docutils.nodes.option_string,
                               docutils.nodes.option_argument)):
            node.pdftext = node.astext()
            if replaceEnt:
                node.pdftext = escape(node.pdftext)

        elif isinstance(node, (docutils.nodes.header, docutils.nodes.footer)):
            node.pdftext = self.gather_pdftext(node, depth)
            if replaceEnt:
                node.pdftext = escape(node.pdftext)

            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, (docutils.nodes.system_message,
                               docutils.nodes.problematic)):
            pre = '<font color="red">'
            post = "</font>"
            node.pdftext = self.gather_pdftext(node, depth)
            if replaceEnt:
                node.pdftext = escape(node.pdftext)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, docutils.nodes.generated):
            node.pdftext = self.gather_pdftext(node, depth)
            if replaceEnt:
                node.pdftext = escape(node.pdftext)
            node.pdftext = pre + node.pdftext + post

        elif isinstance(node, docutils.nodes.image):
            # First see if the image file exists, or else,
            # use image-missing.png
            uri=node.get('uri')
            if not os.path.exists(uri):
                log.error("Missing image file: %s"%uri)
                uri=os.path.join(self.img_dir, 'image-missing.png')
                w, h = 1*cm, 1*cm
            else:
                w, h, kind = self.size_for_image_node(node)
            alignment=node.get('align', 'CENTER').lower()
            if alignment in ('top', 'middle', 'bottom'):
                align='valign="%s"'%alignment
            else:
                align=''

            node.pdftext = '<img src="%s" width="%f" height="%f" %s/>'%\
                (uri, w, h, align)

        elif isinstance(node, math_node):
            mf = Math(node.math_data)
            w, h = mf.wrap(0, 0)
            descent = mf.descent()
            img = mf.genImage()
            self.to_unlink.append(img)
            node.pdftext = '<img src="%s" width=%f height=%f valign=%f/>' % (
                img, w, h, -descent)

        elif isinstance(node, docutils.nodes.footnote_reference):
            # TODO: when used in Sphinx, all footnotes are autonumbered
            anchors=''
            for i in node['ids']:
                if i not in self.targets:
                    anchors+='<a name="%s"/>' % i
                    self.targets.append(i)
            node.pdftext = u'%s<super><a href="%s" color="%s">%s</a></super>'%\
                (anchors, '#' + node.astext(),
                 self.styles.linkColor, node.astext())
                 
        elif isinstance(node, docutils.nodes.citation_reference):
            anchors=''
            for i in node['ids']:
                if i not in self.targets:
                    anchors +='<a name="%s"/>' % i
                    self.targets.append(i)
            node.pdftext = u'%s[<a href="%s" color="%s">%s</a>]'%\
                (anchors, '#' + node.astext(),
                 self.styles.linkColor, node.astext())

        elif isinstance(node, docutils.nodes.target):
            if node['ids'][0] not in self.targets:
                pre = u'<a name="%s"/>' % node['ids'][0]
                self.targets.append(node['ids'][0])
            node.pdftext = self.gather_pdftext(node, depth)
            if replaceEnt:
                node.pdftext = escape(node.pdftext)
            node.pdftext = pre + node.pdftext

        elif isinstance(node, docutils.nodes.inline):
            ftag = self.styleToFont(node['classes'][0])
            if ftag:
                node.pdftext = "%s%s</font>"%\
                    (ftag, self.gather_pdftext(node, depth))
            else:
                node.pdftext = self.gather_pdftext(node, depth)
                
        elif isinstance(node,docutils.nodes.literal_block):
            node.pdftext = self.gather_pdftext(node, depth)

            
        else:
            # With sphinx you will get hundreds of these
            #if not HAS_SPHINX:
            log.warning("Unkn. node (self.gen_pdftext): %s",
                    node.__class__)
            try:
                log.debug(node)
            except (UnicodeDecodeError, UnicodeEncodeError):
                log.debug(repr(node))
            node.pdftext = self.gather_pdftext(node, depth)

        try:
            log.debug("self.gen_pdftext: %s" % node.pdftext)
        except UnicodeDecodeError:
            pass
        # Try to be clever about when to use smartypants
        if node.__class__ in (docutils.nodes.paragraph,
                docutils.nodes.block_quote, docutils.nodes.title):
            node.pdftext = smartyPants(node.pdftext, self.smarty)

        return node.pdftext

    def gen_elements(self, node, depth, style=None):

        log.debug("gen_elements: %s", node.__class__)
        try:
            log.debug("gen_elements: %s", node)
        except (UnicodeDecodeError, UnicodeEncodeError):
            log.debug("gen_elements: %r", node)

        try:
            if node['classes'] and node['classes'][0]:
                # FIXME: Supports only one class, sorry ;-)
                if self.styles.StyleSheet.has_key(node['classes'][0]):
                    style = self.styles[node['classes'][0]]
                else:
                    log.info("Unknown class %s, ignoring.",
                        node['classes'][0])
        except TypeError: # Happens when a docutils.node.Text reaches here
            pass

        if style is None or style == self.styles['bodytext']:
            style = self.styles.styleForNode(node)

        if isinstance(node, docutils.nodes.document):
            node.elements = self.gather_elements(node, depth, style=style)

        elif HAS_SPHINX and isinstance(node, (sphinx.addnodes.glossary,
                                              sphinx.addnodes.start_of_file)):
            node.elements = self.gather_elements(node, depth, style=style)
            
        elif HAS_SPHINX and isinstance(node, (sphinx.addnodes.index)):
            try:
                self.pending_targets.append(node['entries'][0][2])
            except IndexError:
                if node['entries']:
                    log.error("Can't process index entry: %s",node['entries'])
            node.elements = []

        elif isinstance(node, math_node):
            node.elements = [Math(node.math_data)]

        #######################
        ## Tables
        #######################

        elif isinstance(node, docutils.nodes.table):
            node.elements = [Spacer(0, self.styles['table'].spaceBefore)] + \
                            self.gather_elements(node, depth, style=style) +\
                            [Spacer(0, self.styles['table'].spaceAfter)]

        elif isinstance(node, docutils.nodes.tgroup):
            rows = []
            colwidths = []
            hasHead = False
            headRows = 0
            for n in node.children:
                if isinstance(n, docutils.nodes.thead):
                    hasHead = True
                    for row in n.children:
                        r = []
                        for cell in row.children:
                            r.append(cell)
                        rows.append(r)
                    headRows = len(rows)
                elif isinstance(n, docutils.nodes.tbody):
                    for row in n.children:
                        r = []
                        for cell in row.children:
                            r.append(cell)
                        rows.append(r)
                elif isinstance(n, docutils.nodes.colspec):
                    colwidths.append(int(n['colwidth']))

            # colWidths are in no specific unit, really. Maybe ems.
            # Convert them to %
            colwidths=map(int, colwidths)
            tot=sum(colwidths)
            colwidths=["%s%%"%((100.*w)/tot) for w in colwidths]

            if 'colWidths' in style.__dict__:
                colwidths[:len(style.colWidths)]=style.colWidths

            spans = self.filltable(rows)

            data = []
            cellStyles = []
            rowids = range(0, len(rows))
            for row, i in zip(rows, rowids):
                r = []
                j = 0
                for cell in row:
                    if isinstance(cell, str):
                        r.append("")
                    else:
                        # I honestly have no idea what the next line does
                        # (<NAME>, May 25th, 2009)
                        ell = self.gather_elements(cell, depth, style=
                            i < headRows and self.styles['table-heading'] \
                            or style)
                        #if len(ell) == 1:
                            # Experiment: if the cell has a single element,
                            # extract its  class and use it for the cell.
                            # That way, you can have cells with specific
                            # background colors, at least.
                            #
                            # Experiment killed ;-)
                            # You can do that and more using table styles now!
                            #try:
                                #cellStyles += \
                                    #self.styles.pStyleToTStyle(ell[0].style,
                                                               #j, i)
                            ## Fix for issue 85: only do it if it has a style.
                            #except AttributeError:
                                #pass
                        r.append(ell)
                    j += 1
                data.append(r)

            st = TableStyle(spans)
            if 'commands' in self.styles['table'].__dict__:
                for cmd in self.styles['table'].commands:
                    st.add(*cmd)
            if 'commands' in style.__dict__:
                for cmd in style.commands:
                    st.add(*cmd)
            for cmd in cellStyles:
                st.add(*cmd)

            if hasHead:
                for cmd in self.styles.tstyleHead(headRows):
                    st.add(*cmd)
            rtr = self.repeat_table_rows

            node.elements = [DelayedTable(data, colwidths, st, rtr)]

        elif isinstance(node, docutils.nodes.title):
            # Special cases: (Not sure this is right ;-)
            if isinstance(node.parent, docutils.nodes.document):
                node.elements = [Paragraph(self.gen_pdftext(node, depth),
                                           self.styles['title'])]
                self.doc_title = unicode(self.gen_pdftext(node, depth)).strip()
            elif isinstance(node.parent, docutils.nodes.topic):
                node.elements = [Paragraph(self.gen_pdftext(node, depth),
                                           self.styles['topic-title'])]
            elif isinstance(node.parent, docutils.nodes.Admonition):
                node.elements = [Paragraph(self.gen_pdftext(node, depth),
                                           self.styles['admonition-title'])]
            elif isinstance(node.parent, docutils.nodes.table):
                node.elements = [Paragraph(self.gen_pdftext(node, depth),
                                           self.styles['table-title'])]
            elif isinstance(node.parent, docutils.nodes.sidebar):
                node.elements = [Paragraph(self.gen_pdftext(node, depth),
                                           self.styles['sidebar-title'])]
            else:
                # Section/Subsection/etc.
                text = self.gen_pdftext(node, depth)
                fch = node.children[0]
                if isinstance(fch, docutils.nodes.generated) and \
                    fch['classes'] == ['sectnum']:
                    snum = fch.astext()
                else:
                    snum = None
                key = node.get('refid')
                maxdepth=4
                if reportlab.Version > '2.1':
                    maxdepth=6
                
                node.elements = [ Heading(text, 
                        self.styles['heading%d'%min(depth, maxdepth)],
                        level=depth-1,
                        label=key,
                        parent_id=(node.parent.get('ids', [None]) or [None])[0]
                        )]
                if depth <= self.breaklevel:
                    node.elements.insert(0, MyPageBreak())

        elif isinstance(node, docutils.nodes.subtitle):
            if isinstance(node.parent, docutils.nodes.sidebar):
                node.elements = [Paragraph(self.gen_pdftext(node, depth),
                    self.styles['sidebar-subtitle'])]
            elif isinstance(node.parent, docutils.nodes.document):
                node.elements = [Paragraph(self.gen_pdftext(node, depth),
                    self.styles['subtitle'])]

        elif HAS_SPHINX and isinstance(node,
                sphinx.addnodes.compact_paragraph):
            node.elements = self.gather_elements(node, depth, style=style)
            
        elif HAS_SPHINX and isinstance(node,sphinx.addnodes.module):
            node.elements = [Reference('module-'+node['modname'])]

        elif isinstance(node, docutils.nodes.paragraph):
            node.elements = [Paragraph(self.gen_pdftext(node, depth), style)]

        elif isinstance(node, docutils.nodes.docinfo):
{'fraction_non_alphanumeric': 0.10963417405135197, 'fraction_numerical': 0.004061576914337651, 'mean_word_length': 1.5598734913479715, 'pattern_counts': {'":': 4, '<': 101, '<?xml version=': 0, '>': 101, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 44, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4505425', 'n_tokens_mistral': 19875, 'n_tokens_neox': 18903, 'n_words': 4755}