<commit_before>//
// Blackfriday Markdown Processor
// Available at http://github.com/russross/blackfriday
//
// Copyright © 2011 <NAME> <<EMAIL>>.
// Distributed under the Simplified BSD License.
// See README.md for details.
//

//
//
// HTML rendering backend
//
//

package blackfriday

import (
	"bytes"
	"fmt"
	"regexp"
	"strconv"
	"strings"
)

// Html renderer configuration options.
const (
	HTML_SKIP_HTML                = 1 << iota // skip preformatted HTML blocks
	HTML_SKIP_STYLE                           // skip embedded <style> elements
	HTML_SKIP_IMAGES                          // skip embedded images
	HTML_SKIP_LINKS                           // skip all links
	HTML_SANITIZE_OUTPUT                      // strip output of everything that's not known to be safe
	HTML_SAFELINK                             // only link to trusted protocols
	HTML_TOC                                  // generate a table of contents
	HTML_OMIT_CONTENTS                        // skip the main contents (for a standalone table of contents)
	HTML_COMPLETE_PAGE                        // generate a complete HTML page
	HTML_GITHUB_BLOCKCODE                     // use github fenced code rendering rules
	HTML_USE_XHTML                            // generate XHTML output instead of HTML
	HTML_USE_SMARTYPANTS                      // enable smart punctuation substitutions
	HTML_SMARTYPANTS_FRACTIONS                // enable smart fractions (with HTML_USE_SMARTYPANTS)
	HTML_SMARTYPANTS_LATEX_DASHES             // enable LaTeX-style dashes (with HTML_USE_SMARTYPANTS)
)

var (
	tags  = []string{
		"b",
		"blockquote",
		"code",
		"del",
		"dd",
		"dl",
		"dt",
		"em",
		"h1",
		"h2",
		"h3",
		"h4",
		"h5",
		"h6",
		"i",
		"kbd",
		"li",
		"ol",
		"p",
		"pre",
		"s",
		"sup",
		"sub",
		"strong",
		"strike",
		"ul",
	}
	urlRe = `((https?|ftp):\/\/|\/)[-A-Za-z0-9+&@#\/%?=~_|!:,.;\(\)]+`
	tagWhitelist = regexp.MustCompile(`^(<\/?(` + strings.Join(tags, "|") + `)>|<(br|hr)\s?\/?>)$`)
	anchorClean = regexp.MustCompile(`^(<a\shref="` + urlRe + `"(\stitle="[^"<>]+")?\s?>|<\/a>)$`)
	imgClean = regexp.MustCompile(`^(<img\ssrc="` + urlRe + `"(\swidth="\d{1,3}")?(\sheight="\d{1,3}")?(\salt="[^"<>]*")?(\stitle="[^"<>]*")?\s?\/?>)$`)
)

// Html is a type that implements the Renderer interface for HTML output.
//
// Do not create this directly, instead use the HtmlRenderer function.
type Html struct {
	flags    int    // HTML_* options
	closeTag string // how to end singleton tags: either " />\n" or ">\n"
	title    string // document title
	css      string // optional css file url (used with HTML_COMPLETE_PAGE)

	// table of contents data
	tocMarker    int
	headerCount  int
	currentLevel int
	toc          *bytes.Buffer

	smartypants *smartypantsRenderer
}

const (<commit_msg>go fmt<commit_after>//
// Blackfriday Markdown Processor
// Available at http://github.com/russross/blackfriday
//
// Copyright © 2011 <NAME> <<EMAIL>>.
// Distributed under the Simplified BSD License.
// See README.md for details.
//

//
//
// HTML rendering backend
//
//

package blackfriday

import (
	"bytes"
	"fmt"
	"regexp"
	"strconv"
	"strings"
)

// Html renderer configuration options.
const (
	HTML_SKIP_HTML                = 1 << iota // skip preformatted HTML blocks
	HTML_SKIP_STYLE                           // skip embedded <style> elements
	HTML_SKIP_IMAGES                          // skip embedded images
	HTML_SKIP_LINKS                           // skip all links
	HTML_SANITIZE_OUTPUT                      // strip output of everything that's not known to be safe
	HTML_SAFELINK                             // only link to trusted protocols
	HTML_TOC                                  // generate a table of contents
	HTML_OMIT_CONTENTS                        // skip the main contents (for a standalone table of contents)
	HTML_COMPLETE_PAGE                        // generate a complete HTML page
	HTML_GITHUB_BLOCKCODE                     // use github fenced code rendering rules
	HTML_USE_XHTML                            // generate XHTML output instead of HTML
	HTML_USE_SMARTYPANTS                      // enable smart punctuation substitutions
	HTML_SMARTYPANTS_FRACTIONS                // enable smart fractions (with HTML_USE_SMARTYPANTS)
	HTML_SMARTYPANTS_LATEX_DASHES             // enable LaTeX-style dashes (with HTML_USE_SMARTYPANTS)
)

var (
	tags = []string{
		"b",
		"blockquote",
		"code",
		"del",
		"dd",
		"dl",
		"dt",
		"em",
		"h1",
		"h2",
		"h3",
		"h4",
		"h5",
		"h6",
		"i",
		"kbd",
		"li",
		"ol",
		"p",
		"pre",
		"s",
		"sup",
		"sub",
		"strong",
		"strike",
		"ul",
	}
	urlRe        = `((https?|ftp):\/\/|\/)[-A-Za-z0-9+&@#\/%?=~_|!:,.;\(\)]+`
	tagWhitelist = regexp.MustCompile(`^(<\/?(` + strings.Join(tags, "|") + `)>|<(br|hr)\s?\/?>)$`)
	anchorClean  = regexp.MustCompile(`^(<a\shref="` + urlRe + `"(\stitle="[^"<>]+")?\s?>|<\/a>)$`)
	imgClean     = regexp.MustCompile(`^(<img\ssrc="` + urlRe + `"(\swidth="\d{1,3}")?(\sheight="\d{1,3}")?(\salt="[^"<>]*")?(\stitle="[^"<>]*")?\s?\/?>)$`)
)

// Html is a type that implements the Renderer interface for HTML output.
//
// Do not create this directly, instead use the HtmlRenderer function.
type Html struct {
	flags    int    // HTML_* options
	closeTag string // how to end singleton tags: either " />\n" or ">\n"
	title    string // document title
	css      string // optional css file url (used with HTML_COMPLETE_PAGE)

	// table of contents data
	tocMarker    int
	headerCount  int
	currentLevel int
	toc          *bytes.Buffer

	smartypants *smartypantsRenderer
}

const (
{'fraction_non_alphanumeric': 0.15279770444763272, 'fraction_numerical': 0.006097560975609756, 'mean_word_length': 2.195988538681948, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2685939', 'n_tokens_mistral': 2034, 'n_tokens_neox': 1882, 'n_words': 481}