<commit_before>// This file provides the EqC type, which represents strings that can be
// compared only for equality.  Unlike Eq, EqC symbols are first canonicalized
// using a program-provided canonicalization function.

package intern

import (
	"fmt"
)

// An EqC is a string that has been interned to an integer after being
// canonicalized using a program-provided transformation function.  EqC
// supports only equality comparisons, not inequality comparisons.  (No checks
// are performed to enforce that property, unfortunately.)
//
// It is strongly recommended that programs alias EqC once for each
// transformation function.  This will help the compiler catch program errors
// if strings interned using different transformation functions are compared.
type EqC Eq

// eqc maintains all the state needed to manipulate EqCs.
var eqc state

// init initializes our global state.
func init() {
	eqc.forgetAll()
}

// PreEqC provides advance notice of a string that will be interned using
// NewSymbolEqC.  A provided function canonicalizes the string.  Batching up a
// large number of PreEqC calls before calling NewSymbolEqC helps avoid
// running out of symbols that are properly comparable with all other symbols.
func PreEqC(s string, f func(string) string) {
	eqc.Lock()
	eqc.pending = append(eqc.pending, f(s))
	eqc.Unlock()
}

// NewEqC maps a string to an EqC symbol.  It guarantees that two strings
// that are equal after being passed through a function f will return the same
// EqC.
func NewEqC(s string, f func(string) string) EqC {
	var err error
	st := &eqc
	st.Lock()
	defer st.Unlock()
	sym, err := st.assignSymbol(s, f, false)
	if err != nil {
		panic(fmt.Sprintf("Internal error: Unexpected error (%s)", err))
	}
	return EqC(sym)
}

// String converts an EqC back to a string.  It panics if given an EqC that was
// not created using NewEqC.
func (s EqC) String() string {
	return eqc.toString(uint64(s), "EqC")
}<commit_msg>Removed useless function PreEqC<commit_after>// This file provides the EqC type, which represents strings that can be
// compared only for equality.  Unlike Eq, EqC symbols are first canonicalized
// using a program-provided canonicalization function.

package intern

import (
	"fmt"
)

// An EqC is a string that has been interned to an integer after being
// canonicalized using a program-provided transformation function.  EqC
// supports only equality comparisons, not inequality comparisons.  (No checks
// are performed to enforce that property, unfortunately.)
//
// It is strongly recommended that programs alias EqC once for each
// transformation function.  This will help the compiler catch program errors
// if strings interned using different transformation functions are compared.
type EqC Eq

// eqc maintains all the state needed to manipulate EqCs.
var eqc state

// init initializes our global state.
func init() {
	eqc.forgetAll()
}

// NewEqC maps a string to an EqC symbol.  It guarantees that two strings
// that are equal after being passed through a function f will return the same
// EqC.
func NewEqC(s string, f func(string) string) EqC {
	var err error
	st := &eqc
	st.Lock()
	defer st.Unlock()
	sym, err := st.assignSymbol(s, f, false)
	if err != nil {
		panic(fmt.Sprintf("Internal error: Unexpected error (%s)", err))
	}
	return EqC(sym)
}

// String converts an EqC back to a string.  It panics if given an EqC that was
// not created using NewEqC.
func (s EqC) String() string {
	return eqc.toString(uint64(s), "EqC")
}
{'fraction_non_alphanumeric': 0.08155118334759054, 'fraction_numerical': 0.0011405759908753922, 'mean_word_length': 3.79890560875513, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2660833', 'n_tokens_mistral': 1004, 'n_tokens_neox': 979, 'n_words': 478}