<commit_before>package md2roff

import (
	"bytes"
	"fmt"
	"io"
	"regexp"
	"strconv"
	"strings"

	"github.com/russross/blackfriday"
)

// https://github.com/russross/blackfriday/blob/v2/markdown.go
const (
	PARSER_EXTENSIONS = blackfriday.NoIntraEmphasis |
		blackfriday.FencedCode |
		blackfriday.SpaceHeadings |
		blackfriday.AutoHeadingIDs |
		blackfriday.DefinitionLists
)

var (
	backslash     = []byte{'\\'}
	enterVar      = []byte("<var>")
	closeVar      = []byte("</var>")
	tilde         = []byte(`\(ti`)
	htmlEscape    = regexp.MustCompile(`<([A-Za-z][A-Za-z0-9_-]*)>`)
	roffEscape    = regexp.MustCompile(`[&\_-]`)
	headingEscape = regexp.MustCompile(`["]`)
	titleRe       = regexp.MustCompile(`(?P<name>[A-Za-z][A-Za-z0-9_-]+)\((?P<num>\d)\) -- (?P<title>.+)`)
)

func escape(src []byte, re *regexp.Regexp) []byte {
	return re.ReplaceAllFunc(src, func(c []byte) []byte {
		return append(backslash, c...)
	})
}

func roffText(src []byte) []byte {
	return bytes.ReplaceAll(escape(src, roffEscape), []byte{'~'}, tilde)<commit_msg>Compatibility with Go < 1.12<commit_after>package md2roff

import (
	"bytes"
	"fmt"
	"io"
	"regexp"
	"strconv"
	"strings"

	"github.com/russross/blackfriday"
)

// https://github.com/russross/blackfriday/blob/v2/markdown.go
const (
	PARSER_EXTENSIONS = blackfriday.NoIntraEmphasis |
		blackfriday.FencedCode |
		blackfriday.SpaceHeadings |
		blackfriday.AutoHeadingIDs |
		blackfriday.DefinitionLists
)

var (
	backslash     = []byte{'\\'}
	enterVar      = []byte("<var>")
	closeVar      = []byte("</var>")
	tilde         = []byte(`\(ti`)
	htmlEscape    = regexp.MustCompile(`<([A-Za-z][A-Za-z0-9_-]*)>`)
	roffEscape    = regexp.MustCompile(`[&\_-]`)
	headingEscape = regexp.MustCompile(`["]`)
	titleRe       = regexp.MustCompile(`(?P<name>[A-Za-z][A-Za-z0-9_-]+)\((?P<num>\d)\) -- (?P<title>.+)`)
)

func escape(src []byte, re *regexp.Regexp) []byte {
	return re.ReplaceAllFunc(src, func(c []byte) []byte {
		return append(backslash, c...)
	})
}

func roffText(src []byte) []byte {
	return bytes.Replace(escape(src, roffEscape), []byte{'~'}, tilde, -1)
{'fraction_non_alphanumeric': 0.21052631578947367, 'fraction_numerical': 0.007655502392344498, 'mean_word_length': 3.9785714285714286, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2805510', 'n_tokens_mistral': 894, 'n_tokens_neox': 848, 'n_words': 121}