<commit_before># -*- coding: utf-8 -*-
"""
Created on Wed Jun 15 16:32:17 2016

@author: elliott
"""

import re


def convert_columbia_html(text):

    conversions = [('italic', 'em'),
                   ('block_quote', 'blockquote'),
                   ('bold', 'strong'),
                   ('underline', 'u'),
                   ('strikethrough', 'strike'),
                   ('superscript', 'sup'),
                   ('subscript', 'sub'),
                   ('heading', 'h3'),
                   ('table', 'pre')]

    for (pattern, replacement) in conversions:
        text = re.sub('<'+pattern+'>', '<'+replacement+'>', text)
        text = re.sub('</'+pattern+'>', '</'+replacement+'>', text)

    # grayed-out page numbers
    text = re.sub('<page_number>', ' <span class="star-pagination">*', text)
    text = re.sub('</page_number>', '</span> ', text)

    # footnotes
    foot_references = re.findall('<footnote_reference>.*?</footnote_reference>', text)

    for ref in foot_references:
        try:
            fnum = re.search('[\*\d]+', ref).group()
        except AttributeError:
            fnum = re.search('\[fn(.+)\]', ref).group(1)
        rep = '<sup id="ref-fn%s"><a href="#fn%s">%s</a></sup>' % (fnum, fnum, fnum)
        text = text.replace(ref, rep)

    foot_numbers = re.findall('<footnote_number>.*?</footnote_number>',text)

    for ref in foot_numbers:
        try:
            fnum = re.search('[\*\d]+', ref).group()
        except:
            i = ref.find('[fn'+3)
            j = ref.find(']'+3)
            fnum = ref[i:j]
        rep = r'<sup id="fn%s"><a href="#ref-fn%s">%s</a></sup>' % (fnum, fnum, fnum)
        text = text.replace(ref, rep)

    # Make nice paragraphs. This replaces double newlines with paragraphs, then
    # nests paragraphs inside blockquotes, rather than vice versa. The former
    # looks good. The latter is bad.
    text = '<p>' + text + '</p>'
    text = re.sub('</blockquote>\s*<blockquote>', '\n\n', text)
    text = re.sub('\n\n', '</p>\n<p>', text)
    text = re.sub('<p>\s*<blockquote>', '<blockquote><p>', text, re.M)
    text = re.sub('</blockquote></p>', '</p></blockquote>', text, re.M)

    return text<commit_msg>Fixes attribute errors on footnote parsing. (2X)<commit_after># -*- coding: utf-8 -*-
"""
Created on Wed Jun 15 16:32:17 2016

@author: elliott
"""

import re


def convert_columbia_html(text):

    conversions = [('italic', 'em'),
                   ('block_quote', 'blockquote'),
                   ('bold', 'strong'),
                   ('underline', 'u'),
                   ('strikethrough', 'strike'),
                   ('superscript', 'sup'),
                   ('subscript', 'sub'),
                   ('heading', 'h3'),
                   ('table', 'pre')]

    for (pattern, replacement) in conversions:
        text = re.sub('<'+pattern+'>', '<'+replacement+'>', text)
        text = re.sub('</'+pattern+'>', '</'+replacement+'>', text)

    # grayed-out page numbers
    text = re.sub('<page_number>', ' <span class="star-pagination">*', text)
    text = re.sub('</page_number>', '</span> ', text)

    # footnotes
    foot_references = re.findall('<footnote_reference>.*?</footnote_reference>', text)

    for ref in foot_references:
        try:
            fnum = re.search('[\*\d]+', ref).group()
        except AttributeError:
            fnum = re.search('\[fn(.+)\]', ref).group(1)
        rep = '<sup id="ref-fn%s"><a href="#fn%s">%s</a></sup>' % (fnum, fnum, fnum)
        text = text.replace(ref, rep)

    foot_numbers = re.findall('<footnote_number>.*?</footnote_number>',text)

    for ref in foot_numbers:
        try:
            fnum = re.search('[\*\d]+', ref).group()
        except:
            fnum = re.search('\[fn(.+)\]', ref).group(1)
        rep = r'<sup id="fn%s"><a href="#ref-fn%s">%s</a></sup>' % (fnum, fnum, fnum)
        text = text.replace(ref, rep)

    # Make nice paragraphs. This replaces double newlines with paragraphs, then
    # nests paragraphs inside blockquotes, rather than vice versa. The former
    # looks good. The latter is bad.
    text = '<p>' + text + '</p>'
    text = re.sub('</blockquote>\s*<blockquote>', '\n\n', text)
    text = re.sub('\n\n', '</p>\n<p>', text)
    text = re.sub('<p>\s*<blockquote>', '<blockquote><p>', text, re.M)
    text = re.sub('</blockquote></p>', '</p></blockquote>', text, re.M)

    return text
{'fraction_non_alphanumeric': 0.21216955332725615, 'fraction_numerical': 0.007748404740200547, 'mean_word_length': 2.3657975460122698, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6100013', 'n_tokens_mistral': 1491, 'n_tokens_neox': 1439, 'n_words': 330}