<commit_before>// Copyright 2019 <NAME>
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Most of what follows is a subtly changed version of github.com/yuin/goldmark/extension/footnote.go

package common

import (
	"bytes"
	"fmt"
	"os"
	"strconv"
	"unicode"

	"github.com/yuin/goldmark"
	"github.com/yuin/goldmark/ast"
	"github.com/yuin/goldmark/parser"
	"github.com/yuin/goldmark/renderer"
	"github.com/yuin/goldmark/renderer/html"
	"github.com/yuin/goldmark/text"
	"github.com/yuin/goldmark/util"
)

// CleanValue will clean a value to make it safe to be an id
// This function is quite different from the original goldmark function
// and more closely matches the output from the shurcooL sanitizer
// In particular Unicode letters and numbers are a lot more than a-zA-Z0-9...
func CleanValue(value []byte) []byte {
	value = bytes.TrimSpace(value)
	rs := bytes.Runes(value)
	result := make([]rune, 0, len(rs))
	needsDash := false
	for _, r := range rs {
		switch {
		case unicode.IsLetter(r) || unicode.IsNumber(r):
			if needsDash && len(result) > 0 {
				result = append(result, '-')
			}<commit_msg>Don't replace underscores in auto-generated IDs in goldmark (#12805)<commit_after>// Copyright 2019 <NAME>
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Most of what follows is a subtly changed version of github.com/yuin/goldmark/extension/footnote.go

package common

import (
	"bytes"
	"fmt"
	"os"
	"strconv"
	"unicode"

	"github.com/yuin/goldmark"
	"github.com/yuin/goldmark/ast"
	"github.com/yuin/goldmark/parser"
	"github.com/yuin/goldmark/renderer"
	"github.com/yuin/goldmark/renderer/html"
	"github.com/yuin/goldmark/text"
	"github.com/yuin/goldmark/util"
)

// CleanValue will clean a value to make it safe to be an id
// This function is quite different from the original goldmark function
// and more closely matches the output from the shurcooL sanitizer
// In particular Unicode letters and numbers are a lot more than a-zA-Z0-9...
func CleanValue(value []byte) []byte {
	value = bytes.TrimSpace(value)
	rs := bytes.Runes(value)
	result := make([]rune, 0, len(rs))
	needsDash := false
	for _, r := range rs {
		switch {
		case unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_':
			if needsDash && len(result) > 0 {
				result = append(result, '-')
			}
{'fraction_non_alphanumeric': 0.12305854241338113, 'fraction_numerical': 0.011549183592194345, 'mean_word_length': 3.739622641509434, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5195870', 'n_tokens_mistral': 921, 'n_tokens_neox': 837, 'n_words': 269}