<commit_before>#!/usr/bin/python3

import os, re
from markdown import markdown

from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter

def processPygments(body):
    # TODO accept several spaces in regex
    HIGHLIGHT = re.compile("{% ?highlight (\w+) ?%}")
    ENDHIGHLIGHT = re.compile("{% ?endhighlight ?%}")

    result = ""
    bufferedResult = ""
    pygmentsBlock = ""

    for line in body.splitlines():
        if(pygmentsBlock != ""):
            if(ENDHIGHLIGHT.match(line) != None):
                lexer = get_lexer_by_name(pygmentsBlock, stripall=True)
                formatter = HtmlFormatter(linenos=True, cssclass="source")
                result += highlight(bufferedResult, lexer, formatter) + "\n"
                bufferedResult = ""
                pygmentsBlock  = ""
            else:
                bufferedResult += line + "\n"
            continue

        if(HIGHLIGHT.match(line) != None):
            pygmentsBlock = HIGHLIGHT.match(line).groups()[0]
            continue

        result += line + "\n"

    if(pygmentsBlock != ""):
        return None

    return result


def getParsed(rawFile, baseurl):
    """ Return a parsed form of passed page file.

    Passed file should have the following format:

    ""
    ---
    HEADER, 0->* lines of "key: value" entries.
    ---

    BODY, 0->* lines of markdown content.
    ""

    Returned file is represented by a dictionnary containing following entries:
     * "body": a string containing the markdown body converted to HTML
     * the header's content (nothing if file has no header)

     For example, following file

     ""
     ---
     key: value
     ---

     bodyContent
     ""

     would be returned as {"key": "value", "body": "bodyContent"}

     If there's no body part, an empty string is returned in "body".

     If file is bad formatted, None is returned.
     """

    result = {}
    parsedLines = ""
    splittedFile = rawFile.splitlines()

    firstBodyLine = False
    headerPart = False

    if(splittedFile[0] == "---"):
        # First line of passed file declares a header.
        splittedFile.pop(0)
        headerPart = True
    else:
        # File has no header. Directly parse body.
        result["header"] = []
        firstBodyLine = True

    for line in splittedFile:
        if(firstBodyLine):
            # line is the first line of body part.
            firstBodyLine = False
            if(line == ""):
                # First line of the body is empty. Ignore it.
                continue

        if(line == "---" and headerPart):
            # Header's end. Save content of header in result["header"] and
            # reinitialize parsedLines for body
            headerPart = False
            firstBodyLine = True
            result["header"] = parsedLines
            parsedLines = ""
            continue

        line = line.replace("@BASEURL", baseurl)
        parsedLines += (line + "\n")

    parsedLines = processPygments(parsedLines)

    if((headerPart and "header" not in result.keys()) or parsedLines == None):
        # Header was declared, but never closed. Something gone wrong.
        return None

    result["body"] = markdown(parsedLines)

    return result
<commit_msg>Fix regex in juliet/FileParser so multiple spaces are still matched<commit_after>#!/usr/bin/python3

import os, re
from markdown import markdown

from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter

def processPygments(body):
    HIGHLIGHT = re.compile("{%\s*?highlight (\w+)\s*?%}")
    ENDHIGHLIGHT = re.compile("{%\s*?endhighlight\s*?%}")

    result = ""
    bufferedResult = ""
    pygmentsBlock = ""

    for line in body.splitlines():
        if(pygmentsBlock != ""):
            if(ENDHIGHLIGHT.match(line) != None):
                lexer = get_lexer_by_name(pygmentsBlock, stripall=True)
                formatter = HtmlFormatter(linenos=True, cssclass="source")
                result += highlight(bufferedResult, lexer, formatter) + "\n"
                bufferedResult = ""
                pygmentsBlock  = ""
            else:
                bufferedResult += line + "\n"
            continue

        if(HIGHLIGHT.match(line) != None):
            pygmentsBlock = HIGHLIGHT.match(line).groups()[0]
            continue

        result += line + "\n"

    if(pygmentsBlock != ""):
        return None

    return result


def getParsed(rawFile, baseurl):
    """ Return a parsed form of passed page file.

    Passed file should have the following format:

    ""
    ---
    HEADER, 0->* lines of "key: value" entries.
    ---

    BODY, 0->* lines of markdown content.
    ""

    Returned file is represented by a dictionnary containing following entries:
     * "body": a string containing the markdown body converted to HTML
     * the header's content (nothing if file has no header)

     For example, following file

     ""
     ---
     key: value
     ---

     bodyContent
     ""

     would be returned as {"key": "value", "body": "bodyContent"}

     If there's no body part, an empty string is returned in "body".

     If file is bad formatted, None is returned.
     """

    result = {}
    parsedLines = ""
    splittedFile = rawFile.splitlines()

    firstBodyLine = False
    headerPart = False

    if(splittedFile[0] == "---"):
        # First line of passed file declares a header.
        splittedFile.pop(0)
        headerPart = True
    else:
        # File has no header. Directly parse body.
        result["header"] = []
        firstBodyLine = True

    for line in splittedFile:
        if(firstBodyLine):
            # line is the first line of body part.
            firstBodyLine = False
            if(line == ""):
                # First line of the body is empty. Ignore it.
                continue

        if(line == "---" and headerPart):
            # Header's end. Save content of header in result["header"] and
            # reinitialize parsedLines for body
            headerPart = False
            firstBodyLine = True
            result["header"] = parsedLines
            parsedLines = ""
            continue

        line = line.replace("@BASEURL", baseurl)
        parsedLines += (line + "\n")

    parsedLines = processPygments(parsedLines)

    if((headerPart and "header" not in result.keys()) or parsedLines == None):
        # Header was declared, but never closed. Something gone wrong.
        return None

    result["body"] = markdown(parsedLines)

    return result

{'fraction_non_alphanumeric': 0.10468986384266263, 'fraction_numerical': 0.001815431164901664, 'mean_word_length': 2.089252336448598, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3265334', 'n_tokens_mistral': 1908, 'n_tokens_neox': 1790, 'n_words': 569}