<commit_before>package structure

// Set is a map with string key and bool value
// It is not thread safe and modeled after https://github.com/deckarep/golang-set/blob/master/threadunsafe.go
type Set map[string]bool

// NewSet return a pointer of a set using arguments passed to the function
func NewSet(args ...string) *Set {
	// TODO: would this length for the map?
	m := make(Set, len(args))
	for _, v := range args {
		m[v] = true
	}
	return &m
}

// Contains check if a key is presented in the map, it does NOT check the bool value
func (set *Set) Contains(key string) bool {
	_, ok := (*set)[key]
	return ok
}<commit_msg>[common][structure] Add `Add` method to Set<commit_after>package structure

// Set is a map with string key and bool value
// It is not thread safe and modeled after https://github.com/deckarep/golang-set/blob/master/threadunsafe.go
type Set map[string]bool

// NewSet return a pointer of a set using arguments passed to the function
func NewSet(args ...string) *Set {
	// TODO: would this length for the map?
	m := make(Set, len(args))
	for _, v := range args {
		m[v] = true
	}
	return &m
}

// Contains check if a key is presented in the map, it does NOT check the bool value
func (set *Set) Contains(key string) bool {
	_, ok := (*set)[key]
	return ok
}

func (set *Set) Add(key string) {
	(*set)[key] = true
}
{'fraction_non_alphanumeric': 0.11558538404175989, 'fraction_numerical': 0.0, 'mean_word_length': 3.595890410958904, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4355098', 'n_tokens_mistral': 439, 'n_tokens_neox': 426, 'n_words': 184}