<commit_before>package ansiart2utf8

import (
	"fmt"
	"strconv"
	"strings"
)

const (
	SGR_BOLD uint32 = 1 << iota
	SGR_FAINT
	SGR_ITALIC
	SGR_UNDERLINE
	SGR_BLNK_SLOW
	SGR_BLNK_FAST
	SGR_INVERSE
	SGR_CONCEAL
	SGR_STRIKETHROUGH
)

const (
	CIX_FG = iota
	CIX_BG
	CIX_MAX
)

const (
	DEFAULT_FG int = 37
	DEFAULT_BG int = 40
)

type SGR struct {
	// Bold, Faint, Italic, Underline, Blink, Inverse, Conceal, Strikethrough bool
	Flags uint32
	Color [CIX_MAX][]int
}

func (pS *SGR) Fset(f uint32) {
	pS.Flags |= f
}

func (pS *SGR) Fclr(f uint32) {
	pS.Flags &^= f
}

func IaEqual(A, B []int) bool {

	if len(A) != len(B) {
		return false
	}

	for ix := range A {
		if A[ix] != B[ix] {
			return false
		}
	}

	return true
}

func (pS *SGR) ToEsc(pPrev *SGR, bAsDiff, bFakeEscape bool) string {

	// TODO: brighten bold colors?
	sParts := []int{}

	bsIter := []struct {
		Flag  uint32
		Set   int
		Clear int
	}{
		{SGR_BOLD, 1, 22},
		{SGR_FAINT, 2, 22},
		{SGR_ITALIC, 3, 23},
		{SGR_UNDERLINE, 4, 24},
		{SGR_BLNK_SLOW, 5, 25},
		{SGR_BLNK_FAST, 6, 25},
		{SGR_INVERSE, 7, 27},
		{SGR_CONCEAL, 8, 28},
		{SGR_STRIKETHROUGH, 9, 29},
	}

	// APPEND ANSI CODES FOR TEXT STYLE
	flagDiff := pS.Flags ^ pPrev.Flags
	for _, sITER := range bsIter {

		// CLEAR
		if bAsDiff && ((sITER.Flag & flagDiff) != 0) {
			if (sITER.Flag & pS.Flags) != 0 {
				sParts = append(sParts, sITER.Set)
			} else {
				sParts = append(sParts, sITER.Clear)
			}
		}

		if (sITER.Flag & pS.Flags) != 0 {
			sParts = append(sParts, sITER.Set)
		}
	}

	// APPEND ANSI CODES FOR FG/BG COLORS
	mColor := map[int]int{
		CIX_FG: DEFAULT_FG,
		CIX_BG: DEFAULT_BG,
	}

	for CIX := range []int{CIX_FG, CIX_BG} {

		if bAsDiff && IaEqual(pS.Color[CIX], pPrev.Color[CIX]) {<commit_msg>- bugfix to SGR differential encoding<commit_after>package ansiart2utf8

import (
	"fmt"
	"strconv"
	"strings"
)

const (
	SGR_BOLD uint32 = 1 << iota
	SGR_FAINT
	SGR_ITALIC
	SGR_UNDERLINE
	SGR_BLNK_SLOW
	SGR_BLNK_FAST
	SGR_INVERSE
	SGR_CONCEAL
	SGR_STRIKETHROUGH
)

const (
	CIX_FG = iota
	CIX_BG
	CIX_MAX
)

const (
	DEFAULT_FG int = 37
	DEFAULT_BG int = 40
)

type SGR struct {
	// Bold, Faint, Italic, Underline, Blink, Inverse, Conceal, Strikethrough bool
	Flags uint32
	Color [CIX_MAX][]int
}

func (pS *SGR) Fset(f uint32) {
	pS.Flags |= f
}

func (pS *SGR) Fclr(f uint32) {
	pS.Flags &^= f
}

func IaEqual(A, B []int) bool {

	if len(A) != len(B) {
		return false
	}

	for ix := range A {
		if A[ix] != B[ix] {
			return false
		}
	}

	return true
}

func (pS *SGR) ToEsc(pPrev *SGR, bAsDiff, bFakeEscape bool) string {

	// TODO: brighten bold colors?
	sParts := []int{}

	bsIter := []struct {
		Flag  uint32
		Set   int
		Clear int
	}{
		{SGR_BOLD, 1, 22},
		{SGR_FAINT, 2, 22},
		{SGR_ITALIC, 3, 23},
		{SGR_UNDERLINE, 4, 24},
		{SGR_BLNK_SLOW, 5, 25},
		{SGR_BLNK_FAST, 6, 25},
		{SGR_INVERSE, 7, 27},
		{SGR_CONCEAL, 8, 28},
		{SGR_STRIKETHROUGH, 9, 29},
	}

	flagDiff := pS.Flags ^ pPrev.Flags

	// APPEND ANSI CODES FOR TEXT STYLE
	for _, sITER := range bsIter {

		if bAsDiff {

			// NOTE flagDiff (A XOR B) HAS 1s WHERE ATTRIBUTES DIFFER
			if (sITER.Flag & flagDiff) != 0 {

				if (sITER.Flag & pS.Flags) != 0 {
					sParts = append(sParts, sITER.Set)
				} else {
					sParts = append(sParts, sITER.Clear)
				}
			}

		} else if (sITER.Flag & pS.Flags) != 0 {

			sParts = append(sParts, sITER.Set)
		}
	}

	// APPEND ANSI CODES FOR FG/BG COLORS
	mColor := map[int]int{
		CIX_FG: DEFAULT_FG,
		CIX_BG: DEFAULT_BG,
	}

	for CIX := range []int{CIX_FG, CIX_BG} {

		if bAsDiff && IaEqual(pS.Color[CIX], pPrev.Color[CIX]) {
{'fraction_non_alphanumeric': 0.1407469342251951, 'fraction_numerical': 0.026477146042363432, 'mean_word_length': 3.1205510907003444, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6967818', 'n_tokens_mistral': 1925, 'n_tokens_neox': 1683, 'n_words': 396}