<commit_before>//
// Black Friday Markdown Processor
// Originally based on http://github.com/tanoku/upskirt
// by <NAME> <<EMAIL>>
//

//
//
// HTML rendering backend
//
//

package blackfriday

import (
	"bytes"
	"fmt"
	"strconv"
)

const (
	HTML_SKIP_HTML = 1 << iota
	HTML_SKIP_STYLE
	HTML_SKIP_IMAGES
	HTML_SKIP_LINKS
	HTML_EXPAND_TABS
	HTML_SAFELINK
	HTML_TOC
	HTML_HARD_WRAP
	HTML_GITHUB_BLOCKCODE
	HTML_USE_XHTML
	HTML_USE_SMARTYPANTS
	HTML_SMARTYPANTS_FRACTIONS
	HTML_SMARTYPANTS_LATEX_DASHES
)

type htmlOptions struct {
	flags    int
	closeTag string // how to end singleton tags: usually " />\n", possibly ">\n"
	tocData  struct {
		headerCount  int
		currentLevel int
	}
	smartypants *SmartypantsRenderer
}

var xhtmlClose = " />\n"
var htmlClose = ">\n"

func HtmlRenderer(flags int) *Renderer {
	// configure the rendering engine
	r := new(Renderer)
	if flags&HTML_GITHUB_BLOCKCODE == 0 {
		r.BlockCode = htmlBlockCode
	} else {
		r.BlockCode = htmlBlockCodeGithub
	}
	r.BlockQuote = htmlBlockQuote
	if flags&HTML_SKIP_HTML == 0 {
		r.BlockHtml = htmlRawBlock
	}
	r.Header = htmlHeader
	r.HRule = htmlHRule
	r.List = htmlList
	r.ListItem = htmlListItem
	r.Paragraph = htmlParagraph
	r.Table = htmlTable
	r.TableRow = htmlTableRow
	r.TableCell = htmlTableCell

	r.AutoLink = htmlAutoLink
	r.CodeSpan = htmlCodeSpan
	r.DoubleEmphasis = htmlDoubleEmphasis
	r.Emphasis = htmlEmphasis
	if flags&HTML_SKIP_IMAGES == 0 {
		r.Image = htmlImage
	}
	r.LineBreak = htmlLineBreak
	if flags&HTML_SKIP_LINKS == 0 {
		r.Link = htmlLink
	}
	r.RawHtmlTag = htmlRawTag
	r.TripleEmphasis = htmlTripleEmphasis
	r.StrikeThrough = htmlStrikeThrough

	var cb *SmartypantsRenderer
	if flags&HTML_USE_SMARTYPANTS == 0 {
		r.NormalText = htmlNormalText
	} else {
		cb = Smartypants(flags)
		r.NormalText = htmlSmartypants
	}

	closeTag := htmlClose
	if flags&HTML_USE_XHTML != 0 {
		closeTag = xhtmlClose
	}
	r.Opaque = &htmlOptions{flags: flags, closeTag: closeTag, smartypants: cb}
	return r
}

func HtmlTocRenderer(flags int) *Renderer {
	// configure the rendering engine
	r := new(Renderer)
	r.Header = htmlTocHeader

	r.CodeSpan = htmlCodeSpan
	r.DoubleEmphasis = htmlDoubleEmphasis
	r.Emphasis = htmlEmphasis
	r.TripleEmphasis = htmlTripleEmphasis
	r.StrikeThrough = htmlStrikeThrough

	r.DocumentFooter = htmlTocFinalize

	closeTag := ">\n"
	if flags&HTML_USE_XHTML != 0 {
		closeTag = " />\n"
	}
	r.Opaque = &htmlOptions{flags: flags | HTML_TOC, closeTag: closeTag}
	return r
}

func attrEscape(out *bytes.Buffer, src []byte) {
	org := 0
	for i, ch := range src {
		switch ch {
		case '<':
			if i > org {
				// copy all the normal characters since the last escape
				out.Write(src[org:i])
			}
			org = i + 1
			out.WriteString("&lt;")
		case '>':
			if i > org {
				out.Write(src[org:i])
			}
			org = i + 1
			out.WriteString("&gt;")
		case '&':
			if i > org {
				out.Write(src[org:i])
			}<commit_msg>dumb tweak that gives a little speed bump<commit_after>//
// Black Friday Markdown Processor
// Originally based on http://github.com/tanoku/upskirt
// by <NAME> <<EMAIL>>
//

//
//
// HTML rendering backend
//
//

package blackfriday

import (
	"bytes"
	"fmt"
	"strconv"
)

const (
	HTML_SKIP_HTML = 1 << iota
	HTML_SKIP_STYLE
	HTML_SKIP_IMAGES
	HTML_SKIP_LINKS
	HTML_EXPAND_TABS
	HTML_SAFELINK
	HTML_TOC
	HTML_HARD_WRAP
	HTML_GITHUB_BLOCKCODE
	HTML_USE_XHTML
	HTML_USE_SMARTYPANTS
	HTML_SMARTYPANTS_FRACTIONS
	HTML_SMARTYPANTS_LATEX_DASHES
)

type htmlOptions struct {
	flags    int
	closeTag string // how to end singleton tags: usually " />\n", possibly ">\n"
	tocData  struct {
		headerCount  int
		currentLevel int
	}
	smartypants *SmartypantsRenderer
}

var xhtmlClose = " />\n"
var htmlClose = ">\n"

func HtmlRenderer(flags int) *Renderer {
	// configure the rendering engine
	r := new(Renderer)
	if flags&HTML_GITHUB_BLOCKCODE == 0 {
		r.BlockCode = htmlBlockCode
	} else {
		r.BlockCode = htmlBlockCodeGithub
	}
	r.BlockQuote = htmlBlockQuote
	if flags&HTML_SKIP_HTML == 0 {
		r.BlockHtml = htmlRawBlock
	}
	r.Header = htmlHeader
	r.HRule = htmlHRule
	r.List = htmlList
	r.ListItem = htmlListItem
	r.Paragraph = htmlParagraph
	r.Table = htmlTable
	r.TableRow = htmlTableRow
	r.TableCell = htmlTableCell

	r.AutoLink = htmlAutoLink
	r.CodeSpan = htmlCodeSpan
	r.DoubleEmphasis = htmlDoubleEmphasis
	r.Emphasis = htmlEmphasis
	if flags&HTML_SKIP_IMAGES == 0 {
		r.Image = htmlImage
	}
	r.LineBreak = htmlLineBreak
	if flags&HTML_SKIP_LINKS == 0 {
		r.Link = htmlLink
	}
	r.RawHtmlTag = htmlRawTag
	r.TripleEmphasis = htmlTripleEmphasis
	r.StrikeThrough = htmlStrikeThrough

	var cb *SmartypantsRenderer
	if flags&HTML_USE_SMARTYPANTS == 0 {
		r.NormalText = htmlNormalText
	} else {
		cb = Smartypants(flags)
		r.NormalText = htmlSmartypants
	}

	closeTag := htmlClose
	if flags&HTML_USE_XHTML != 0 {
		closeTag = xhtmlClose
	}
	r.Opaque = &htmlOptions{flags: flags, closeTag: closeTag, smartypants: cb}
	return r
}

func HtmlTocRenderer(flags int) *Renderer {
	// configure the rendering engine
	r := new(Renderer)
	r.Header = htmlTocHeader

	r.CodeSpan = htmlCodeSpan
	r.DoubleEmphasis = htmlDoubleEmphasis
	r.Emphasis = htmlEmphasis
	r.TripleEmphasis = htmlTripleEmphasis
	r.StrikeThrough = htmlStrikeThrough

	r.DocumentFooter = htmlTocFinalize

	closeTag := ">\n"
	if flags&HTML_USE_XHTML != 0 {
		closeTag = " />\n"
	}
	r.Opaque = &htmlOptions{flags: flags | HTML_TOC, closeTag: closeTag}
	return r
}

func attrEscape(out *bytes.Buffer, src []byte) {
	org := 0
	for i, ch := range src {
		// doing this check is a bit faster than falling into
		// the switch statement. as the compiler improves, this
		// should be unnecessary
		if ch != '"' && ch != '&' && ch != '<' && ch != '>' {
			continue
		}

		switch ch {
		case '<':
			if i > org {
				// copy all the normal characters since the last escape
				out.Write(src[org:i])
			}
			org = i + 1
			out.WriteString("&lt;")
		case '>':
			if i > org {
				out.Write(src[org:i])
			}
			org = i + 1
			out.WriteString("&gt;")
		case '&':
			if i > org {
				out.Write(src[org:i])
			}
{'fraction_non_alphanumeric': 0.10894684714427204, 'fraction_numerical': 0.0036315615714757345, 'mean_word_length': 3.6251908396946564, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4339311', 'n_tokens_mistral': 2513, 'n_tokens_neox': 2285, 'n_words': 530}