<commit_before>from django.utils.safestring import mark_safe
#from markdown import markdown
from django import template
import re
import pypandoc
from django.contrib.staticfiles.templatetags import staticfiles
from django.utils import timezone
from datetime import timedelta

register = template.Library()

InlineMathPat = re.compile(r'\\\((.+?)\\\)', flags=re.DOTALL)
DisplayMathPat = re.compile(r'\\\[(.+?)\\\]', flags=re.DOTALL)
StaticImagePat = re.compile(r'STATICIMAGE/([^"]+)')

@register.filter(name='md2html')
def md2html(txt, stripP=False):
    'converst ReST to HTML using pandoc, w/ audio support'
    txt, markers = add_temporary_markers(txt, find_audio)
    txt, videoMarkers = add_temporary_markers(txt, find_video, len(markers))
    try:
        txt = pypandoc.convert(txt, 'html', format='rst',
                               extra_args=('--mathjax',))
    except StandardError:
        pass
    txt = replace_temporary_markers(txt, audio_html, markers)
    txt = replace_temporary_markers(txt, video_html, videoMarkers)
    txt = StaticImagePat.sub(staticfiles.static('ct/') + r'\1', txt)
    if stripP and txt.startswith('<p>') and txt.endswith('</p>'):
        txt = txt[3:-4]
    return mark_safe(txt)

def nolongerused():
    'convert markdown to html, preserving latex delimiters'
    # markdown replaces \( with (, so have to protect our math...
    # replace \(math\) with \\(math\\)
    txt = InlineMathPat.sub(r'\\\\(\1\\\\)', txt)
    # replace \[math\] with \\[math\\]
    txt = DisplayMathPat.sub(r'\\\\[\1\\\\]', txt)
    txt = markdown(txt, safe_mode='escape')
    if stripP and txt.startswith('<p>') and txt.endswith('</p>'):
        txt = txt[3:-4]
    return mark_safe(txt)

def find_audio(txt, lastpos, tag='.. audio::'):
        i = txt.find(tag, lastpos)
        if i < 0:
            return -1, None, None
        lastpos = i + len(tag)
        k = txt.find('\n', lastpos)
        if k < 0:
            k = txt.find('\r', lastpos)
        if k < 0: # no EOL, slurp to end of text
            k = len(txt)<commit_msg>fix(ct): fix static ur for S3 storage<commit_after>from django.utils.safestring import mark_safe
#from markdown import markdown
from django import template
import re
import pypandoc
from django.contrib.staticfiles.templatetags import staticfiles
from django.utils import timezone
from datetime import timedelta

register = template.Library()

InlineMathPat = re.compile(r'\\\((.+?)\\\)', flags=re.DOTALL)
DisplayMathPat = re.compile(r'\\\[(.+?)\\\]', flags=re.DOTALL)
StaticImagePat = re.compile(r'STATICIMAGE/([^"]+)')

@register.filter(name='md2html')
def md2html(txt, stripP=False):
    'converst ReST to HTML using pandoc, w/ audio support'
    txt, markers = add_temporary_markers(txt, find_audio)
    txt, videoMarkers = add_temporary_markers(txt, find_video, len(markers))
    try:
        txt = pypandoc.convert(txt, 'html', format='rst',
                               extra_args=('--mathjax',))
    except StandardError:
        pass
    txt = replace_temporary_markers(txt, audio_html, markers)
    txt = replace_temporary_markers(txt, video_html, videoMarkers)
    txt = StaticImagePat.sub(staticfiles.static('ct') + '/' + r'\1', txt)
    if stripP and txt.startswith('<p>') and txt.endswith('</p>'):
        txt = txt[3:-4]
    return mark_safe(txt)

def nolongerused():
    'convert markdown to html, preserving latex delimiters'
    # markdown replaces \( with (, so have to protect our math...
    # replace \(math\) with \\(math\\)
    txt = InlineMathPat.sub(r'\\\\(\1\\\\)', txt)
    # replace \[math\] with \\[math\\]
    txt = DisplayMathPat.sub(r'\\\\[\1\\\\]', txt)
    txt = markdown(txt, safe_mode='escape')
    if stripP and txt.startswith('<p>') and txt.endswith('</p>'):
        txt = txt[3:-4]
    return mark_safe(txt)

def find_audio(txt, lastpos, tag='.. audio::'):
        i = txt.find(tag, lastpos)
        if i < 0:
            return -1, None, None
        lastpos = i + len(tag)
        k = txt.find('\n', lastpos)
        if k < 0:
            k = txt.find('\r', lastpos)
        if k < 0: # no EOL, slurp to end of text
            k = len(txt)
{'fraction_non_alphanumeric': 0.15808556925308195, 'fraction_numerical': 0.006526468455402465, 'mean_word_length': 2.9372026641294005, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5198378', 'n_tokens_mistral': 1431, 'n_tokens_neox': 1389, 'n_words': 355}