<commit_before>"""this file encapsulates the bibtex handling routines.
Only in a separate file because elpy crashed on bibtexparser import :-(
"""

import bibtexparser
import re
import os

def processBibtex(docname, bibtex):
    """process a raw bibtex input as downloaded from wiki.

    Main steps:
    - remove any wiki anchors
    - sanitize via bibtex library
    - write out bibtex file
    - return a list of bibtex keys, necessary for postprocessing latex

    input:
    - docname: document and subdirectory to be processed
    - bibtex: string containing the collated files

    output:
    - list of bibtex keys
    """

    # remove any wiki anchors, as defined in
    # https://meta.wikimedia.org/wiki/Help:Anchors

    bibtex = re.sub('<( *)div(.*?)>', '', bibtex, re.S)
    bibtex = re.sub('<( *)/( *)div(.*?)>', '', bibtex, re.S)

    # sanitize via bibtex library
    parser = bibtexparser.bparser.BibTexParser()
    parser.customization = bibtexparser.customization.homogeneize_latex_encoding
    bibDB = bibtexparser.loads(bibtex,
                               parser=parser)

    # lowercase all the bibtex keys, just because that's the way pandoc inserts the refs:
    for e in bibDB.entries:
        e['ID'] = e['ID'].lower()


    # dump it to file
    writer = bibtexparser.bwriter.BibTexWriter()
    with open (os.path.join(docname, 'tex', 'bib.bib'),
               'w') as bh:
        bh.write(writer.write(bibDB))

    # get a list keys to return:
    keys = [x['ID'] for x in bibDB.entries]
    print "bibtexkeys (1): ", keys
    return keys
<commit_msg>whitespace fix<commit_after>"""this file encapsulates the bibtex handling routines.
Only in a separate file because elpy crashed on bibtexparser import :-(
"""

import bibtexparser
import re
import os


def processBibtex(docname, bibtex):
    """process a raw bibtex input as downloaded from wiki.

    Main steps:
    - remove any wiki anchors
    - sanitize via bibtex library
    - write out bibtex file
    - return a list of bibtex keys, necessary for postprocessing latex

    input:
    - docname: document and subdirectory to be processed
    - bibtex: string containing the collated files

    output:
    - list of bibtex keys
    """

    # remove any wiki anchors, as defined in
    # https://meta.wikimedia.org/wiki/Help:Anchors

    bibtex = re.sub('<( *)div(.*?)>', '', bibtex, re.S)
    bibtex = re.sub('<( *)/( *)div(.*?)>', '', bibtex, re.S)

    # sanitize via bibtex library
    parser = bibtexparser.bparser.BibTexParser()
    parser.customization = bibtexparser.customization.homogeneize_latex_encoding
    bibDB = bibtexparser.loads(bibtex,
                               parser=parser)

    # lowercase all the bibtex keys, just because that's the way pandoc inserts the refs:
    for e in bibDB.entries:
        e['ID'] = e['ID'].lower()


    # dump it to file
    writer = bibtexparser.bwriter.BibTexWriter()
    with open (os.path.join(docname, 'tex', 'bib.bib'),
               'w') as bh:
        bh.write(writer.write(bibDB))

    # get a list keys to return:
    keys = [x['ID'] for x in bibDB.entries]
    print "bibtexkeys (1): ", keys
    return keys

{'fraction_non_alphanumeric': 0.11041009463722397, 'fraction_numerical': 0.0006309148264984228, 'mean_word_length': 2.7794994040524434, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2590995', 'n_tokens_mistral': 1051, 'n_tokens_neox': 949, 'n_words': 323}