<commit_before>__all__ = ['writer']

import docutils.core as dc
import docutils.writers
from docutils import nodes

from docutils.writers.latex2e import (Writer, LaTeXTranslator,
                                      PreambleCmds)

from rstmath import mathEnv
import code_block

from options import options

try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

PreambleCmds.float_settings = '''
\\usepackage[font={small,it},labelfont=bf]{caption}
\\usepackage{float}
'''

class Translator(LaTeXTranslator):
    def __init__(self, *args, **kwargs):
        LaTeXTranslator.__init__(self, *args, **kwargs)

        # Handle author declarations

        self.current_field = ''

        self.author_names = []
        self.author_institutions = []
        self.author_institution_map = dict()
        self.author_emails = []
        self.paper_title = ''
        self.abstract_text = []
        self.keywords = ''
        self.table_caption = []
        self.video_url = ''
        self.bibliography = ''

        # This gets read by the underlying docutils implementation.
        # If present, it is a list with the first entry the style name
        # and the second entry the BiBTeX file (see `visit_field_body`)
        self.bibtex = None

        self.abstract_in_progress = False
        self.non_breaking_paragraph = False

        self.figure_type = 'figure'
        self.figure_alignment = 'left'
        self.table_type = 'table'

        self.active_table.set_table_style('booktabs')

    def visit_docinfo(self, node):
        pass

    def depart_docinfo(self, node):
        pass

    def visit_author(self, node):
        self.author_names.append(self.encode(node.astext()))
        self.author_institution_map[self.author_names[-1]] = []
        raise nodes.SkipNode

    def depart_author(self, node):
        pass

    def visit_author(self, node):
        self.author_names.append(self.encode(node.astext()))
        self.author_institution_map[self.author_names[-1]] = []
        raise nodes.SkipNode

    def depart_author(self, node):
        pass

    def visit_classifier(self, node):
        pass

    def depart_classifier(self, node):
        pass

    def visit_field_name(self, node):
        self.current_field = node.astext()
        raise nodes.SkipNode

    def visit_field_body(self, node):
        text = self.encode(node.astext())

        if self.current_field == 'email':
            self.author_emails.append(text)
        elif self.current_field == 'institution':
            self.author_institutions.append(text)
            self.author_institution_map[self.author_names[-1]].append(text)
        elif self.current_field == 'video':
            self.video_url = text
        elif self.current_field == 'bibliography':
            self.bibtex = ['alpha', text]
            self._use_latex_citations = True
            self._bibitems = ['', '']

        self.current_field = ''

        raise nodes.SkipNode

    def depart_field_body(self, node):
        raise nodes.SkipNode

    def depart_document(self, node):
        LaTeXTranslator.depart_document(self, node)

        ## Generate footmarks

        # build map: institution -> (author1, author2)
        institution_authors = OrderedDict()
        for auth in self.author_institution_map:
            for inst in self.author_institution_map[auth]:
                institution_authors.setdefault(inst, []).append(auth)

        def footmark(n):
            """Insert footmark #n.  Footmark 1 is reserved for
            the corresponding author.\
            """
            return ('\\setcounter{footnotecounter}{%d}' % n,
                    '\\fnsymbol{footnotecounter}')

        # Build one footmark for each institution
        institute_footmark = {}
        for i, inst in enumerate(institution_authors):
            institute_footmark[inst] = footmark(i + 2)

        footmark_template = r'\thanks{%(footmark)s %(instutions)}'
        corresponding_auth_template = r'''%%
          %(footmark_counter)s\thanks{%(footmark)s %%
          Corresponding author: \protect\href{mailto:%(email)s}{%(email)s}}'''

        title = self.paper_title
        authors = []
        institutions_mentioned = set()
        for n, auth in enumerate(self.author_names):
            # Corresponding author
            if n == 0:
                authors += [r'%(author)s$^{%(footmark)s}$' % \
                            {'author': auth,
                             'footmark': ''.join(footmark(1)) + ''.join([''.join(institute_footmark[inst]) for inst in self.author_institution_map[auth]])}]

                fm_counter, fm = footmark(1)
                authors[-1] += corresponding_auth_template % \
                               {'footmark_counter': fm_counter,
                                'footmark': fm,
                                'email': self.author_emails[0]}

            else:
                authors += [r'%(author)s$^{%(footmark)s}$' %
                            {'author': auth,
                             'footmark': ''.join([''.join(institute_footmark[inst]) for inst in self.author_institution_map[auth]])}]

            for inst in self.author_institution_map[auth]:
                if not inst in institutions_mentioned:
                    fm_counter, fm = institute_footmark[inst]
                    authors[-1] += r'%(footmark_counter)s\thanks{%(footmark)s %(institution)s}' % \
                                {'footmark_counter': fm_counter,
                                 'footmark': fm,
                                 'institution': inst}

                institutions_mentioned.add(inst)


        ## Add copyright

        # If things went spectacularly wrong, we could not even parse author
        # info.  Just fill in some dummy info so that we can see the error
        # messages in the resulting PDF.
        if len(self.author_names) == 0:
            self.author_names = ['<NAME>']
            self.author_emails = ['<EMAIL>']
            authors = ['']

        copyright_holder = self.author_names[0] + ('.' if len(self.author_names) == 1 else ' et al.')
        author_notes = r'''%%

          \noindent%%
          Copyright\,\copyright\,%(year)s %(copyright_holder)s %(copyright)s%%
        ''' % \
        {'email': self.author_emails[0],
         'year': options['proceedings']['year'],
         'copyright_holder': copyright_holder,
         'copyright': options['proceedings']['copyright']['article']}

        authors[-1] += r'\thanks{%s}' % author_notes


        ## Set up title and page headers

        if not self.video_url:
            video_template = ''
        else:
            video_template = r'\\\vspace{5mm}\tt\url{%s}\vspace{-5mm}' % self.video_url

        title_template = r'\newcounter{footnotecounter}' \
                r'\title{%s}\author{%s' \
                r'%s}\maketitle'
        title_template = title_template % (title, ', '.join(authors),
                                           video_template)

        marks = r'''
          \renewcommand{\leftmark}{%s}
          \renewcommand{\rightmark}{%s}
        ''' % (options['proceedings']['title']['short'], title.upper())
        title_template += marks

        self.body_pre_docinfo = [title_template]

        # Save paper stats
        self.document.stats = {'title': title,
                               'authors': ', '.join(self.author_names),
                               'author': self.author_names,
                               'author_email': self.author_emails,
                               'author_institution': self.author_institutions,
                               'abstract': self.abstract_text,
                               'keywords': self.keywords,
                               'copyright_holder': copyright_holder,
                               'video': self.video_url}

        if hasattr(self, 'bibtex') and self.bibtex:
            self.document.stats.update({'bibliography': self.bibtex[1]})

    def end_open_abstract(self, node):
        if 'abstract' not in node['classes'] and self.abstract_in_progress:
            self.out.append('\\end{abstract}')
            self.abstract_in_progress = False
        elif self.abstract_in_progress:
            self.abstract_text.append(self.encode(node.astext()))


    def visit_title(self, node):
        self.end_open_abstract(node)

        if self.section_level == 1:
            if self.paper_title:
                import warnings
                warnings.warn(RuntimeWarning("Title set twice--ignored. "
                                             "Could be due to ReST"
                                             "error.)"))
            else:
                self.paper_title = self.encode(node.astext())
            raise nodes.SkipNode

        elif node.astext() == 'References':
            raise nodes.SkipNode

        LaTeXTranslator.visit_title(self, node)
<commit_msg>added bibliogrpahy key to json config record, '' as default/null value<commit_after>__all__ = ['writer']

import docutils.core as dc
import docutils.writers
from docutils import nodes

from docutils.writers.latex2e import (Writer, LaTeXTranslator,
                                      PreambleCmds)

from rstmath import mathEnv
import code_block

from options import options

try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

PreambleCmds.float_settings = '''
\\usepackage[font={small,it},labelfont=bf]{caption}
\\usepackage{float}
'''

class Translator(LaTeXTranslator):
    def __init__(self, *args, **kwargs):
        LaTeXTranslator.__init__(self, *args, **kwargs)

        # Handle author declarations

        self.current_field = ''

        self.author_names = []
        self.author_institutions = []
        self.author_institution_map = dict()
        self.author_emails = []
        self.paper_title = ''
        self.abstract_text = []
        self.keywords = ''
        self.table_caption = []
        self.video_url = ''
        self.bibliography = ''

        # This gets read by the underlying docutils implementation.
        # If present, it is a list with the first entry the style name
        # and the second entry the BiBTeX file (see `visit_field_body`)
        self.bibtex = None

        self.abstract_in_progress = False
        self.non_breaking_paragraph = False

        self.figure_type = 'figure'
        self.figure_alignment = 'left'
        self.table_type = 'table'

        self.active_table.set_table_style('booktabs')

    def visit_docinfo(self, node):
        pass

    def depart_docinfo(self, node):
        pass

    def visit_author(self, node):
        self.author_names.append(self.encode(node.astext()))
        self.author_institution_map[self.author_names[-1]] = []
        raise nodes.SkipNode

    def depart_author(self, node):
        pass

    def visit_author(self, node):
        self.author_names.append(self.encode(node.astext()))
        self.author_institution_map[self.author_names[-1]] = []
        raise nodes.SkipNode

    def depart_author(self, node):
        pass

    def visit_classifier(self, node):
        pass

    def depart_classifier(self, node):
        pass

    def visit_field_name(self, node):
        self.current_field = node.astext()
        raise nodes.SkipNode

    def visit_field_body(self, node):
        text = self.encode(node.astext())

        if self.current_field == 'email':
            self.author_emails.append(text)
        elif self.current_field == 'institution':
            self.author_institutions.append(text)
            self.author_institution_map[self.author_names[-1]].append(text)
        elif self.current_field == 'video':
            self.video_url = text
        elif self.current_field == 'bibliography':
            self.bibtex = ['alpha', text]
            self._use_latex_citations = True
            self._bibitems = ['', '']
            self.bibliography = text

        self.current_field = ''

        raise nodes.SkipNode

    def depart_field_body(self, node):
        raise nodes.SkipNode

    def depart_document(self, node):
        LaTeXTranslator.depart_document(self, node)

        ## Generate footmarks

        # build map: institution -> (author1, author2)
        institution_authors = OrderedDict()
        for auth in self.author_institution_map:
            for inst in self.author_institution_map[auth]:
                institution_authors.setdefault(inst, []).append(auth)

        def footmark(n):
            """Insert footmark #n.  Footmark 1 is reserved for
            the corresponding author.\
            """
            return ('\\setcounter{footnotecounter}{%d}' % n,
                    '\\fnsymbol{footnotecounter}')

        # Build one footmark for each institution
        institute_footmark = {}
        for i, inst in enumerate(institution_authors):
            institute_footmark[inst] = footmark(i + 2)

        footmark_template = r'\thanks{%(footmark)s %(instutions)}'
        corresponding_auth_template = r'''%%
          %(footmark_counter)s\thanks{%(footmark)s %%
          Corresponding author: \protect\href{mailto:%(email)s}{%(email)s}}'''

        title = self.paper_title
        authors = []
        institutions_mentioned = set()
        for n, auth in enumerate(self.author_names):
            # Corresponding author
            if n == 0:
                authors += [r'%(author)s$^{%(footmark)s}$' % \
                            {'author': auth,
                             'footmark': ''.join(footmark(1)) + ''.join([''.join(institute_footmark[inst]) for inst in self.author_institution_map[auth]])}]

                fm_counter, fm = footmark(1)
                authors[-1] += corresponding_auth_template % \
                               {'footmark_counter': fm_counter,
                                'footmark': fm,
                                'email': self.author_emails[0]}

            else:
                authors += [r'%(author)s$^{%(footmark)s}$' %
                            {'author': auth,
                             'footmark': ''.join([''.join(institute_footmark[inst]) for inst in self.author_institution_map[auth]])}]

            for inst in self.author_institution_map[auth]:
                if not inst in institutions_mentioned:
                    fm_counter, fm = institute_footmark[inst]
                    authors[-1] += r'%(footmark_counter)s\thanks{%(footmark)s %(institution)s}' % \
                                {'footmark_counter': fm_counter,
                                 'footmark': fm,
                                 'institution': inst}

                institutions_mentioned.add(inst)


        ## Add copyright

        # If things went spectacularly wrong, we could not even parse author
        # info.  Just fill in some dummy info so that we can see the error
        # messages in the resulting PDF.
        if len(self.author_names) == 0:
            self.author_names = ['<NAME>']
            self.author_emails = ['<EMAIL>']
            authors = ['']

        copyright_holder = self.author_names[0] + ('.' if len(self.author_names) == 1 else ' et al.')
        author_notes = r'''%%

          \noindent%%
          Copyright\,\copyright\,%(year)s %(copyright_holder)s %(copyright)s%%
        ''' % \
        {'email': self.author_emails[0],
         'year': options['proceedings']['year'],
         'copyright_holder': copyright_holder,
         'copyright': options['proceedings']['copyright']['article']}

        authors[-1] += r'\thanks{%s}' % author_notes


        ## Set up title and page headers

        if not self.video_url:
            video_template = ''
        else:
            video_template = r'\\\vspace{5mm}\tt\url{%s}\vspace{-5mm}' % self.video_url

        title_template = r'\newcounter{footnotecounter}' \
                r'\title{%s}\author{%s' \
                r'%s}\maketitle'
        title_template = title_template % (title, ', '.join(authors),
                                           video_template)

        marks = r'''
          \renewcommand{\leftmark}{%s}
          \renewcommand{\rightmark}{%s}
        ''' % (options['proceedings']['title']['short'], title.upper())
        title_template += marks

        self.body_pre_docinfo = [title_template]

        # Save paper stats
        self.document.stats = {'title': title,
                               'authors': ', '.join(self.author_names),
                               'author': self.author_names,
                               'author_email': self.author_emails,
                               'author_institution': self.author_institutions,
                               'abstract': self.abstract_text,
                               'keywords': self.keywords,
                               'copyright_holder': copyright_holder,
                               'video': self.video_url,
                               'bibliography':self.bibliography}

        if hasattr(self, 'bibtex') and self.bibtex:
            self.document.stats.update({'bibliography': self.bibtex[1]})

    def end_open_abstract(self, node):
        if 'abstract' not in node['classes'] and self.abstract_in_progress:
            self.out.append('\\end{abstract}')
            self.abstract_in_progress = False
        elif self.abstract_in_progress:
            self.abstract_text.append(self.encode(node.astext()))


    def visit_title(self, node):
        self.end_open_abstract(node)

        if self.section_level == 1:
            if self.paper_title:
                import warnings
                warnings.warn(RuntimeWarning("Title set twice--ignored. "
                                             "Could be due to ReST"
                                             "error.)"))
            else:
                self.paper_title = self.encode(node.astext())
            raise nodes.SkipNode

        elif node.astext() == 'References':
            raise nodes.SkipNode

        LaTeXTranslator.visit_title(self, node)

{'fraction_non_alphanumeric': 0.11411328276396655, 'fraction_numerical': 0.002546924312053596, 'mean_word_length': 1.909003060074086, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3281876', 'n_tokens_mistral': 5125, 'n_tokens_neox': 4832, 'n_words': 1012}