<commit_before>package flag

import (
	"go/ast"
	"reflect"
	"strings"
	"unicode"
)

type register struct {
}

var defaultRegister register

func (r register) addIndexes(indexes map[string]int, keys []string, index int) {
	for _, key := range keys {
		indexes[key] = index
	}
}

func (r register) findDuplicates(parent, set *FlagSet, names []string) []string {
	var duplicates []string
	for _, name := range names {
		if set.isFlagOrSubset(name) || (parent != nil && parent.isFlagOrSubset(name)) {
			duplicates = append(duplicates, name)
			continue
		}
		for i := range set.subsets {
			if set.subsets[i].isFlagOrSubset(name) {
				duplicates = append(duplicates, name)
				break
			}
		}
	}
	return duplicates
}

const (
	flagNameSeparatorForSplit = ","
	flagNameSeparatorForJoin  = ", "
)

func (r register) joinFlagNames(names []string) string {
	return strings.Join(names, flagNameSeparatorForJoin)
}

func (r register) cleanFlagNames(names string) ([]string, string) {
	ns := splitAndTrimSpace(names, flagNameSeparatorForSplit)
	return ns, r.joinFlagNames(ns)
}

func (r register) cleanFlag(flag *Flag) {
	if flag.ValSep == "" {
		flag.ValSep = ","
	}
	r.updateFlagDesc(flag, flag.Desc)
	r.updateFlagVersion(flag, flag.Version)
}

func (r register) updateFlagDefault(flag *Flag, def interface{}) error {
	refPtr := reflect.ValueOf(flag.Ptr)
	var compatible bool

	refdef := reflect.ValueOf(def)
	if isRefvalSlicePtr(refPtr) {
		compatible = refdef.Kind() == reflect.Slice
		compatible = compatible && isKindCompatible(sliceElemKind(refPtr.Elem()), sliceElemKind(refdef))
	} else {
		compatible = isKindCompatible(refPtr.Elem().Kind(), refdef.Kind())
	}
	if !compatible {
		return newErrorf(errInvalidType, "incompatible default value type: %s", flag.Names)
	}
	flag.Default = def
	return nil
}

func (r register) updateFlagSelects(flag *Flag, val interface{}) error {
	if val == nil {
		return nil
	}

	refval := reflect.ValueOf(flag.Ptr).Elem()
	k := sliceElemKind(refval)
	if isKindNumber(k) {
		fs := convertNumbersToFloats(val)
		flag.Selects = fs
		return nil
	}
	if k == reflect.String {
		if vals, ok := val.([]string); ok && len(vals) != 0 {
			flag.Selects = vals
			return nil
		}
	}
	return newErrorf(errInvalidSelects, "invalid selects: %s, %v", flag.Names, val)
}

func (r register) registerFlag(parent, set *FlagSet, flag Flag) error {
	refval := reflect.ValueOf(flag.Ptr)
	if refval.Kind() != reflect.Ptr {
		return newErrorf(errNonPointer, "illegal flag pointer: %s", flag.Names)
	}
	if typeName(flag.Ptr) == "" {
		return newErrorf(errInvalidType, "unsupported flag type: %s", flag.Names)
	}
	if flag.Default != nil {
		err := r.updateFlagDefault(&flag, flag.Default)
		if err != nil {
			return err
		}
	}
	if flag.Selects != nil {
		err := r.updateFlagSelects(&flag, flag.Selects)
		if err != nil {
			return err
		}
	}

	ns, names := r.cleanFlagNames(flag.Names)
	if duplicates := r.findDuplicates(parent, set, ns); len(duplicates) > 0 {
		return newErrorf(errDuplicateFlagRegister, "duplicate flags with parent/self/childs: %s->%s, %v", parent.self.Names, set.self.Names, duplicates)
	}

	flag.Names = names
	r.cleanFlag(&flag)

	set.flags = append(set.flags, flag)
	r.addIndexes(set.flagIndexes, ns, len(set.flags)-1)
	return nil
}

func (r register) checkSubsetValid(flag *Flag) error {
	if flag.Names == "" {
		return newErrorf(errInvalidNames, "subset names should not be empty")
	}
	return nil
}

func (r register) registerSet(parent, set *FlagSet, flag Flag) (*FlagSet, error) {
	var ns []string

	ns, flag.Names = r.cleanFlagNames(flag.Names)
	err := r.checkSubsetValid(&flag)
	if err != nil {
		return nil, err
	}

	if duplicates := r.findDuplicates(parent, set, ns); len(duplicates) > 0 {
		return nil, newErrorf(errDuplicateFlagRegister, "duplicate subset name: %v", duplicates)
	}<commit_msg>fix error message printing for duplicated flags.<commit_after>package flag

import (
	"go/ast"
	"reflect"
	"strings"
	"unicode"
)

type register struct {
}

var defaultRegister register

func (r register) addIndexes(indexes map[string]int, keys []string, index int) {
	for _, key := range keys {
		indexes[key] = index
	}
}

func (r register) findDuplicates(parent, set *FlagSet, names []string) []string {
	var duplicates []string
	for _, name := range names {
		if set.isFlagOrSubset(name) || (parent != nil && parent.isFlagOrSubset(name)) {
			duplicates = append(duplicates, name)
			continue
		}
		for i := range set.subsets {
			if set.subsets[i].isFlagOrSubset(name) {
				duplicates = append(duplicates, name)
				break
			}
		}
	}
	return duplicates
}

const (
	flagNameSeparatorForSplit = ","
	flagNameSeparatorForJoin  = ", "
)

func (r register) joinFlagNames(names []string) string {
	return strings.Join(names, flagNameSeparatorForJoin)
}

func (r register) cleanFlagNames(names string) ([]string, string) {
	ns := splitAndTrimSpace(names, flagNameSeparatorForSplit)
	return ns, r.joinFlagNames(ns)
}

func (r register) cleanFlag(flag *Flag) {
	if flag.ValSep == "" {
		flag.ValSep = ","
	}
	r.updateFlagDesc(flag, flag.Desc)
	r.updateFlagVersion(flag, flag.Version)
}

func (r register) updateFlagDefault(flag *Flag, def interface{}) error {
	refPtr := reflect.ValueOf(flag.Ptr)
	var compatible bool

	refdef := reflect.ValueOf(def)
	if isRefvalSlicePtr(refPtr) {
		compatible = refdef.Kind() == reflect.Slice
		compatible = compatible && isKindCompatible(sliceElemKind(refPtr.Elem()), sliceElemKind(refdef))
	} else {
		compatible = isKindCompatible(refPtr.Elem().Kind(), refdef.Kind())
	}
	if !compatible {
		return newErrorf(errInvalidType, "incompatible default value type: %s", flag.Names)
	}
	flag.Default = def
	return nil
}

func (r register) updateFlagSelects(flag *Flag, val interface{}) error {
	if val == nil {
		return nil
	}

	refval := reflect.ValueOf(flag.Ptr).Elem()
	k := sliceElemKind(refval)
	if isKindNumber(k) {
		fs := convertNumbersToFloats(val)
		flag.Selects = fs
		return nil
	}
	if k == reflect.String {
		if vals, ok := val.([]string); ok && len(vals) != 0 {
			flag.Selects = vals
			return nil
		}
	}
	return newErrorf(errInvalidSelects, "invalid selects: %s, %v", flag.Names, val)
}

func (r register) registerFlag(parent, set *FlagSet, flag Flag) error {
	refval := reflect.ValueOf(flag.Ptr)
	if refval.Kind() != reflect.Ptr {
		return newErrorf(errNonPointer, "illegal flag pointer: %s", flag.Names)
	}
	if typeName(flag.Ptr) == "" {
		return newErrorf(errInvalidType, "unsupported flag type: %s", flag.Names)
	}
	if flag.Default != nil {
		err := r.updateFlagDefault(&flag, flag.Default)
		if err != nil {
			return err
		}
	}
	if flag.Selects != nil {
		err := r.updateFlagSelects(&flag, flag.Selects)
		if err != nil {
			return err
		}
	}

	ns, names := r.cleanFlagNames(flag.Names)
	if duplicates := r.findDuplicates(parent, set, ns); len(duplicates) > 0 {
		if parent != nil {
			return newErrorf(errDuplicateFlagRegister, "duplicate flags with parent/self/children: %s->%s, %v", parent.self.Names, set.self.Names, duplicates)
		}
		return newErrorf(errDuplicateFlagRegister, "duplicate flags with self/children: %s, %v", set.self.Names, duplicates)
	}

	flag.Names = names
	r.cleanFlag(&flag)

	set.flags = append(set.flags, flag)
	r.addIndexes(set.flagIndexes, ns, len(set.flags)-1)
	return nil
}

func (r register) checkSubsetValid(flag *Flag) error {
	if flag.Names == "" {
		return newErrorf(errInvalidNames, "subset names should not be empty")
	}
	return nil
}

func (r register) registerSet(parent, set *FlagSet, flag Flag) (*FlagSet, error) {
	var ns []string

	ns, flag.Names = r.cleanFlagNames(flag.Names)
	err := r.checkSubsetValid(&flag)
	if err != nil {
		return nil, err
	}

	if duplicates := r.findDuplicates(parent, set, ns); len(duplicates) > 0 {
		return nil, newErrorf(errDuplicateFlagRegister, "duplicate subset name: %v", duplicates)
	}
{'fraction_non_alphanumeric': 0.13784748788574344, 'fraction_numerical': 0.00102014792144861, 'mean_word_length': 4.40896551724138, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3247369', 'n_tokens_mistral': 2880, 'n_tokens_neox': 2727, 'n_words': 683}