<commit_before>package decor

import (
	"fmt"
	"io"
	"math"
	"strconv"
	"strings"
)

const (
	_ib   SizeB1024 = iota + 1
	_iKiB SizeB1024 = 1 << (iota * 10)
	_iMiB
	_iGiB
	_iTiB
)

//go:generate stringer -type=SizeB1024 -trimprefix=_i
type SizeB1024 int64

func (self SizeB1024) Format(st fmt.State, verb rune) {
	var prec int
	switch verb {
	case 'd':
	case 's':
		prec = -1
	default:
		if p, ok := st.Precision(); ok {
			prec = p
		} else {
			prec = 6
		}
	}

	var unit SizeB1024
	switch {
	case self < _iKiB:
		unit = _ib
	case self < _iMiB:
		unit = _iKiB
	case self < _iGiB:
		unit = _iMiB
	case self < _iTiB:
		unit = _iGiB
	case self <= math.MaxInt64:
		unit = _iTiB
	}

	var b strings.Builder
	b.WriteString(strconv.FormatFloat(float64(self)/float64(unit), 'f', prec, 64))

	if st.Flag(' ') {
		b.WriteString(" ")
	}
	b.WriteString(unit.String())

	if w, ok := st.Width(); ok {
		if l := b.Len(); l < w {
			pad := strings.Repeat(" ", w-l)
			if st.Flag('-') {
				b.WriteString(pad)
			} else {
				tmp := b.String()
				b.Reset()
				b.WriteString(pad)
				b.WriteString(tmp)
			}
		}
	}

	io.WriteString(st, b.String())
}

const (
	_b  SizeB1000 = 1
	_KB SizeB1000 = 1000
	_MB SizeB1000 = _KB * 1000<commit_msg>for consistency sake<commit_after>package decor

import (
	"fmt"
	"io"
	"math"
	"strconv"
	"strings"
)

const (
	_ib   SizeB1024 = iota + 1
	_iKiB SizeB1024 = 1 << (iota * 10)
	_iMiB
	_iGiB
	_iTiB
)

//go:generate stringer -type=SizeB1024 -trimprefix=_i
type SizeB1024 int64

func (self SizeB1024) Format(st fmt.State, verb rune) {
	var prec int
	switch verb {
	case 'd':
	case 's':
		prec = -1
	default:
		if p, ok := st.Precision(); ok {
			prec = p
		} else {
			prec = 6
		}
	}

	var unit SizeB1024
	switch {
	case self < _iKiB:
		unit = _ib
	case self < _iMiB:
		unit = _iKiB
	case self < _iGiB:
		unit = _iMiB
	case self < _iTiB:
		unit = _iGiB
	case self <= math.MaxInt64:
		unit = _iTiB
	}

	var b strings.Builder
	b.WriteString(strconv.FormatFloat(float64(self)/float64(unit), 'f', prec, 64))

	if st.Flag(' ') {
		b.WriteString(" ")
	}
	b.WriteString(unit.String())

	if w, ok := st.Width(); ok {
		if l := b.Len(); l < w {
			pad := strings.Repeat(" ", w-l)
			if st.Flag('-') {
				b.WriteString(pad)
			} else {
				tmp := b.String()
				b.Reset()
				b.WriteString(pad)
				b.WriteString(tmp)
			}
		}
	}

	io.WriteString(st, b.String())
}

const (
	_b  SizeB1000 = 1
	_KB SizeB1000 = _b * 1000
	_MB SizeB1000 = _KB * 1000
{'fraction_non_alphanumeric': 0.15, 'fraction_numerical': 0.04959349593495935, 'mean_word_length': 2.745814307458143, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3347963', 'n_tokens_mistral': 1292, 'n_tokens_neox': 1079, 'n_words': 266}