<commit_before>package config

import "fmt"

// ReplaceObjects will traverse through the config object and replace all
// objects that has a the given '$' + key property with the value returned
// from replacement(obj).
//
// This is useful when implementing TransformationProviders.
func ReplaceObjects(
	config map[string]interface{},
	key string,
	replacement func(obj map[string]interface{}) (interface{}, error),
) error {
	_, err := inject(key, replacement, config)
	return err
}

func inject(
	key string,
	replacement func(obj map[string]interface{}) (interface{}, error),
	val interface{},
) (interface{}, error) {
	switch val := val.(type) {
	case []interface{}:
		for i, v := range val {
			v2, err := inject(key, replacement, v)
			if err != nil {
				return nil, err
			}
			val[i] = v2
		}
	case map[string]interface{}:
		if _, ok := val["$"+key].(string); ok {
			return replacement(val)
		}
		for k, v := range val {
			v2, err := inject(key, replacement, v)
			if err != nil {
				return nil, err
			}
			val[k] = v2
		}
	}
	return val, nil
}

func jsonCompatTypes(val interface{}) error {
	switch val := val.(type) {
	case map[string]interface{}:
		for _, v := range val {
			if err := jsonCompatTypes(v); err != nil {
				return err
			}
		}
	case []interface{}:
		for _, v := range val {
			if err := jsonCompatTypes(v); err != nil {
				return err<commit_msg>Allow replacement property values to be non-strings<commit_after>package config

import "fmt"

// ReplaceObjects will traverse through the config object and replace all
// objects that has a the given '$' + key property with the value returned
// from replacement(obj).
//
// This is useful when implementing TransformationProviders.
func ReplaceObjects(
	config map[string]interface{},
	key string,
	replacement func(obj map[string]interface{}) (interface{}, error),
) error {
	_, err := inject(key, replacement, config)
	return err
}

func inject(
	key string,
	replacement func(obj map[string]interface{}) (interface{}, error),
	val interface{},
) (interface{}, error) {
	switch val := val.(type) {
	case []interface{}:
		for i, v := range val {
			v2, err := inject(key, replacement, v)
			if err != nil {
				return nil, err
			}
			val[i] = v2
		}
	case map[string]interface{}:
		if _, ok := val["$"+key].(interface{}); ok {
			return replacement(val)
		}
		for k, v := range val {
			v2, err := inject(key, replacement, v)
			if err != nil {
				return nil, err
			}
			val[k] = v2
		}
	}
	return val, nil
}

func jsonCompatTypes(val interface{}) error {
	switch val := val.(type) {
	case map[string]interface{}:
		for _, v := range val {
			if err := jsonCompatTypes(v); err != nil {
				return err
			}
		}
	case []interface{}:
		for _, v := range val {
			if err := jsonCompatTypes(v); err != nil {
				return err
{'fraction_non_alphanumeric': 0.14306100606493044, 'fraction_numerical': 0.0028540849090260435, 'mean_word_length': 3.4792332268370605, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4584600', 'n_tokens_mistral': 988, 'n_tokens_neox': 891, 'n_words': 305}