<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function

import re
import sys

entities = [
    ("&", "&amp;"),
    ("'", "&apos;"),
    ('"', "&quot;"),
    ("<", "&lt;"),
    (">", "&gt;"),
]

patterns = [
    (   re.compile(r"\\\\"),
        lambda match: "<br />"
    ),
    (   re.compile(r"\{\{\{\{(.*?)(\}\}\}\})"),
        lambda match: "<code>{0}</code>".format(*match.groups())
    ),
    (   re.compile(r"\{\{\{(.*?)(\}\}\})"),
        lambda match: "<tt>{0}</tt>".format(*match.groups())
    ),
    (   re.compile(r"\*\*(.*?)(\*\*|$)"),
        lambda match: "<strong>{0}</strong>".format(*match.groups())
    ),
    (   re.compile(r"(?<!:)//(.*?)(?<!:)(//|$)"),
        lambda match: "<em>{0}</em>".format(*match.groups())
    ),
    # superscript, e.g. "E=mc^^2^^" (non-standard)
    (   re.compile(r"\^\^(.*?)(\^\^|$)"),
        lambda match: "<sup>{0}</sup>".format(*match.groups())
    ),
    # subscript, e.g. "H,,2,,SO,,4,," (non-standard)
    (   re.compile(r",,(.*?)(,,|$)"),
        lambda match: "<sub>{0}</sub>".format(*match.groups())
    ),
    (   re.compile(r"\[\[([^\|]*?)(\]\]|$)"),
        lambda match: '<a href="{0}">{0}</a>'.format(*match.groups())
    ),
    (   re.compile(r"\[\[([^\|]*?)\|([^\|]*?)(\]\]|$)"),
        lambda match: '<a href="{0}">{1}</a>'.format(*match.groups())
    ),
]

HEADING = [n * "=" for n in range(7)]
HORIZONTAL_RULE = "----"

class Document(object):

    def __init__(self, markup):
        self.blocks = [[]]
        literal = False
        for raw_line in markup.splitlines(True):
            line = raw_line.strip()
            if line.startswith("="):
                for level in range(6, 0, -1):
                    if line.startswith(HEADING[level]):
                        self.blocks.append([level * "=" + line.strip("=").strip()])
                        break
                self.blocks.append([])
            elif line.startswith(HORIZONTAL_RULE):
                self.blocks.append([HORIZONTAL_RULE])
                self.blocks.append([])
            elif line.startswith("{{{{") and "}}}}" not in line:
                self.blocks.append(["{{{{" + " ".join(line[4:].lstrip().split())])
                self.blocks.append([])
                literal = True
            elif line.startswith("}}}}"):
                self.blocks.append(["}}}}"])
                self.blocks.append([])
                literal = False
            elif line.startswith("{{{") and "}}}" not in line:
                self.blocks.append(["{{{" + " ".join(line[3:].lstrip().split())])
                self.blocks.append([])
                literal = True
            elif line.startswith("}}}"):
                self.blocks.append(["}}}"])
                self.blocks.append([])
                literal = False
            else:
                if literal:
                    if self.blocks[-1]:
                        self.blocks[-1][-1] += raw_line
                    else:
                        self.blocks[-1].append(raw_line)
                elif line:
                    self.blocks[-1].append(line)
                else:
                    self.blocks.append([])
        self.blocks = [" ".join(block) for block in self.blocks if block]
        
    def __xhtml__(self, fragment=True):
        out = []
        literal = False
        for block in self.blocks:
            if block.startswith("="):
                for level in range(6, 0, -1):
                    if block.startswith(HEADING[level]):
                        out.append("<h{0}>{1}</h{0}>".format(level, block[level:]))
                        break
            elif block == HORIZONTAL_RULE:
                out.append("<hr />")
            elif block == "{{{{":
                out.append("<pre><code>")
                literal = True
            elif block.startswith("{{{{"):
                out.append('<pre class="{0}"><code>'.format(block[4:]))
                literal = True
            elif block.startswith("}}}}"):
                out.append("</code></pre>")
                literal = False
            elif block == "{{{":
                out.append("<pre>")
                literal = True
            elif block.startswith("{{{"):
                out.append('<pre class="{0}">'.format(block[3:]))
                literal = True
            elif block.startswith("}}}"):
                out.append("</pre>")
                literal = False
            else:
                for char, entity in entities:
                    block = block.replace(char, entity)
                if literal:
                    out.append(block)
                else:
                    out.append("<p>")
                    for pattern in patterns:
                        block = pattern[0].sub(pattern[1], block)
                    out.append(block)
                    out.append("</p>")
        if fragment:
            return "".join(out)
        else:
            return "<!doctype html>\n<html><head></head><body>{0}</body></html>".format("".join(out))

def to_xhtml(markup, fragment=True):
    out = Document(markup).__xhtml__(fragment)
    #print(out)
    return out

def __test__():
    assert to_xhtml("foo bar") == "<p>foo bar</p>"
    assert to_xhtml("foo\nbar") == "<p>foo bar</p>"
    assert to_xhtml("foo\n\nbar") == "<p>foo</p><p>bar</p>"
    assert to_xhtml("foo\n\n\n\nbar") == "<p>foo</p><p>bar</p>"
    assert to_xhtml("foo & bar") == "<p>foo &amp; bar</p>"
    assert to_xhtml("foo 'bar'") == "<p>foo &apos;bar&apos;</p>"
    assert to_xhtml('foo "bar"') == "<p>foo &quot;bar&quot;</p>"
    assert to_xhtml("foo < bar") == "<p>foo &lt; bar</p>"
    assert to_xhtml("foo > bar") == "<p>foo &gt; bar</p>"
    assert to_xhtml("**foo** bar") == "<p><strong>foo</strong> bar</p>"
    assert to_xhtml("foo //bar//") == "<p>foo <em>bar</em></p>"
    assert to_xhtml("**foo** **bar**") == "<p><strong>foo</strong> <strong>bar</strong></p>"<commit_msg>fixed html namespace<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function

import re
import sys

entities = [
    ("&", "&amp;"),
    ("'", "&apos;"),
    ('"', "&quot;"),
    ("<", "&lt;"),
    (">", "&gt;"),
]

patterns = [
    (   re.compile(r"\\\\"),
        lambda match: "<br />"
    ),
    (   re.compile(r"\{\{\{\{(.*?)(\}\}\}\})"),
        lambda match: "<code>{0}</code>".format(*match.groups())
    ),
    (   re.compile(r"\{\{\{(.*?)(\}\}\})"),
        lambda match: "<tt>{0}</tt>".format(*match.groups())
    ),
    (   re.compile(r"\*\*(.*?)(\*\*|$)"),
        lambda match: "<strong>{0}</strong>".format(*match.groups())
    ),
    (   re.compile(r"(?<!:)//(.*?)(?<!:)(//|$)"),
        lambda match: "<em>{0}</em>".format(*match.groups())
    ),
    # superscript, e.g. "E=mc^^2^^" (non-standard)
    (   re.compile(r"\^\^(.*?)(\^\^|$)"),
        lambda match: "<sup>{0}</sup>".format(*match.groups())
    ),
    # subscript, e.g. "H,,2,,SO,,4,," (non-standard)
    (   re.compile(r",,(.*?)(,,|$)"),
        lambda match: "<sub>{0}</sub>".format(*match.groups())
    ),
    (   re.compile(r"\[\[([^\|]*?)(\]\]|$)"),
        lambda match: '<a href="{0}">{0}</a>'.format(*match.groups())
    ),
    (   re.compile(r"\[\[([^\|]*?)\|([^\|]*?)(\]\]|$)"),
        lambda match: '<a href="{0}">{1}</a>'.format(*match.groups())
    ),
]

HEADING = [n * "=" for n in range(7)]
HORIZONTAL_RULE = "----"

class Document(object):

    def __init__(self, markup):
        self.blocks = [[]]
        literal = False
        for raw_line in markup.splitlines(True):
            line = raw_line.strip()
            if line.startswith("="):
                for level in range(6, 0, -1):
                    if line.startswith(HEADING[level]):
                        self.blocks.append([level * "=" + line.strip("=").strip()])
                        break
                self.blocks.append([])
            elif line.startswith(HORIZONTAL_RULE):
                self.blocks.append([HORIZONTAL_RULE])
                self.blocks.append([])
            elif line.startswith("{{{{") and "}}}}" not in line:
                self.blocks.append(["{{{{" + " ".join(line[4:].lstrip().split())])
                self.blocks.append([])
                literal = True
            elif line.startswith("}}}}"):
                self.blocks.append(["}}}}"])
                self.blocks.append([])
                literal = False
            elif line.startswith("{{{") and "}}}" not in line:
                self.blocks.append(["{{{" + " ".join(line[3:].lstrip().split())])
                self.blocks.append([])
                literal = True
            elif line.startswith("}}}"):
                self.blocks.append(["}}}"])
                self.blocks.append([])
                literal = False
            else:
                if literal:
                    if self.blocks[-1]:
                        self.blocks[-1][-1] += raw_line
                    else:
                        self.blocks[-1].append(raw_line)
                elif line:
                    self.blocks[-1].append(line)
                else:
                    self.blocks.append([])
        self.blocks = [" ".join(block) for block in self.blocks if block]
        
    def __xhtml__(self, fragment=True):
        out = []
        literal = False
        for block in self.blocks:
            if block.startswith("="):
                for level in range(6, 0, -1):
                    if block.startswith(HEADING[level]):
                        out.append("<h{0}>{1}</h{0}>".format(level, block[level:]))
                        break
            elif block == HORIZONTAL_RULE:
                out.append("<hr />")
            elif block == "{{{{":
                out.append("<pre><code>")
                literal = True
            elif block.startswith("{{{{"):
                out.append('<pre class="{0}"><code>'.format(block[4:]))
                literal = True
            elif block.startswith("}}}}"):
                out.append("</code></pre>")
                literal = False
            elif block == "{{{":
                out.append("<pre>")
                literal = True
            elif block.startswith("{{{"):
                out.append('<pre class="{0}">'.format(block[3:]))
                literal = True
            elif block.startswith("}}}"):
                out.append("</pre>")
                literal = False
            else:
                for char, entity in entities:
                    block = block.replace(char, entity)
                if literal:
                    out.append(block)
                else:
                    out.append("<p>")
                    for pattern in patterns:
                        block = pattern[0].sub(pattern[1], block)
                    out.append(block)
                    out.append("</p>")
        if fragment:
            return "".join(out)
        else:
            return "<html xmlns=\"http://www.w3.org/1999/xhtml\"><head></head><body>{0}</body></html>".format("".join(out))

def to_xhtml(markup, fragment=True):
    out = Document(markup).__xhtml__(fragment)
    #print(out)
    return out

def __test__():
    assert to_xhtml("foo bar") == "<p>foo bar</p>"
    assert to_xhtml("foo\nbar") == "<p>foo bar</p>"
    assert to_xhtml("foo\n\nbar") == "<p>foo</p><p>bar</p>"
    assert to_xhtml("foo\n\n\n\nbar") == "<p>foo</p><p>bar</p>"
    assert to_xhtml("foo & bar") == "<p>foo &amp; bar</p>"
    assert to_xhtml("foo 'bar'") == "<p>foo &apos;bar&apos;</p>"
    assert to_xhtml('foo "bar"') == "<p>foo &quot;bar&quot;</p>"
    assert to_xhtml("foo < bar") == "<p>foo &lt; bar</p>"
    assert to_xhtml("foo > bar") == "<p>foo &gt; bar</p>"
    assert to_xhtml("**foo** bar") == "<p><strong>foo</strong> bar</p>"
    assert to_xhtml("foo //bar//") == "<p>foo <em>bar</em></p>"
    assert to_xhtml("**foo** **bar**") == "<p><strong>foo</strong> <strong>bar</strong></p>"
{'fraction_non_alphanumeric': 0.2253757662272231, 'fraction_numerical': 0.00680157863800487, 'mean_word_length': 1.7043596730245232, 'pattern_counts': {'":': 4, '<': 158, '<?xml version=': 0, '>': 154, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3178144', 'n_tokens_mistral': 3855, 'n_tokens_neox': 3706, 'n_words': 637}