<commit_before>package tests

import (
	"fmt"
	"math"
	"reflect"
	"testing"
	"time"

	"github.com/go-graphite/carbonapi/expr/interfaces"
	"github.com/go-graphite/carbonapi/expr/metadata"
	"github.com/go-graphite/carbonapi/expr/types"
	"github.com/go-graphite/carbonapi/pkg/parser"
	pb "github.com/go-graphite/protocol/carbonapi_v3_pb"
)

type FuncEvaluator struct {
	eval func(e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error)
}

func (evaluator *FuncEvaluator) EvalExpr(e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
	if e.IsName() {
		return values[parser.MetricRequest{Metric: e.Target(), From: from, Until: until}], nil
	} else if e.IsConst() {
		p := types.MetricData{FetchResponse: pb.FetchResponse{Name: e.Target(), Values: []float64{e.FloatValue()}}}
		return []*types.MetricData{&p}, nil
	}
	// evaluate the function

	// all functions have arguments -- check we do too
	if len(e.Args()) == 0 {
		return nil, parser.ErrMissingArgument
	}

	return evaluator.eval(e, from, until, values)
}

func EvaluatorFromFunc(function interfaces.Function) interfaces.Evaluator {
	e := &FuncEvaluator{
		eval: function.Do,
	}

	return e
}

func EvaluatorFromFuncWithMetadata(metadata map[string]interfaces.Function) interfaces.Evaluator {
	e := &FuncEvaluator{
		eval: func(e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
			if f, ok := metadata[e.Target()]; ok {
				return f.Do(e, from, until, values)
			}
			return nil, fmt.Errorf("unknown function: %v", e.Target())
		},
	}
	return e
}

func DeepClone(original map[parser.MetricRequest][]*types.MetricData) map[parser.MetricRequest][]*types.MetricData {
	clone := map[parser.MetricRequest][]*types.MetricData{}
	for key, originalMetrics := range original {
		copiedMetrics := []*types.MetricData{}
		for _, originalMetric := range originalMetrics {
			copiedMetric := types.MetricData{
				FetchResponse: pb.FetchResponse{
					Name:      originalMetric.Name,
					StartTime: originalMetric.StartTime,
					StopTime:  originalMetric.StopTime,
					StepTime:  originalMetric.StepTime,
					Values:    make([]float64, len(originalMetric.Values)),
				},
			}

			copy(copiedMetric.Values, originalMetric.Values)
			copiedMetrics = append(copiedMetrics, &copiedMetric)
		}

		clone[key] = copiedMetrics
	}

	return clone
}

func compareFloat64(v1, v2 float64) bool {
	if math.IsNaN(v1) && math.IsNaN(v2) {
		return true
	}
	if math.IsInf(v1, 1) && math.IsInf(v2, 1) {
		return true
	}

	if math.IsInf(v1, 0) && math.IsInf(v2, 0) {
		return true
	}

	d := math.Abs(v1 - v2)
	return d < eps
}

func deepCompareFields(v1, v2 reflect.Value) bool {
	if !v1.CanInterface() {
		return true
	}
	t1 := v1.Type()
	if t1.Comparable() {
		if t1.Name() == "float64" {
			return compareFloat64(v1.Interface().(float64), v2.Interface().(float64))
		}
		if t1.Name() == "float32" {
			v1f64 := float64(v1.Interface().(float32))
			v2f64 := float64(v2.Interface().(float32))
			return compareFloat64(v1f64, v2f64)
		}
		return reflect.DeepEqual(v1.Interface(), v2.Interface())
	} else {
		switch v1.Kind() {
		case reflect.Struct:
			if v1.NumField() == 0 {
				// We don't know how to compare that
				return false
			}
			for i := 0; i < v1.NumField(); i++ {
				r := deepCompareFields(v1.Field(i), v2.Field(i))
				if !r {
					return r
				}
			}
		case reflect.Slice, reflect.Array:
			if v1.Len() != v2.Len() {
				return false
			}
			if v1.Len() == 0 {
				return true
			}
			if v1.Index(0).Kind() != v2.Index(0).Kind() {
				return false
			}
			for i := 0; i < v1.Len(); i++ {
				e1 := v1.Index(i)
				e2 := v2.Index(i)
				return deepCompareFields(e1, e2)
			}
		case reflect.Func:
			return v1.Pointer() == v2.Pointer()
		default:
			fmt.Printf("unsupported v1.Kind=%v t1.Name=%v, t1.Value=%v\n\n", v1.Kind(), v1.Type().Name(), v1.String())
			return false
		}
	}
	return true
}<commit_msg>Fix bug in test helper<commit_after>package tests

import (
	"fmt"
	"math"
	"reflect"
	"testing"
	"time"

	"github.com/go-graphite/carbonapi/expr/interfaces"
	"github.com/go-graphite/carbonapi/expr/metadata"
	"github.com/go-graphite/carbonapi/expr/types"
	"github.com/go-graphite/carbonapi/pkg/parser"
	pb "github.com/go-graphite/protocol/carbonapi_v3_pb"
)

type FuncEvaluator struct {
	eval func(e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error)
}

func (evaluator *FuncEvaluator) EvalExpr(e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
	if e.IsName() {
		return values[parser.MetricRequest{Metric: e.Target(), From: from, Until: until}], nil
	} else if e.IsConst() {
		p := types.MetricData{FetchResponse: pb.FetchResponse{Name: e.Target(), Values: []float64{e.FloatValue()}}}
		return []*types.MetricData{&p}, nil
	}
	// evaluate the function

	// all functions have arguments -- check we do too
	if len(e.Args()) == 0 {
		return nil, parser.ErrMissingArgument
	}

	return evaluator.eval(e, from, until, values)
}

func EvaluatorFromFunc(function interfaces.Function) interfaces.Evaluator {
	e := &FuncEvaluator{
		eval: function.Do,
	}

	return e
}

func EvaluatorFromFuncWithMetadata(metadata map[string]interfaces.Function) interfaces.Evaluator {
	e := &FuncEvaluator{
		eval: func(e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
			if f, ok := metadata[e.Target()]; ok {
				return f.Do(e, from, until, values)
			}
			return nil, fmt.Errorf("unknown function: %v", e.Target())
		},
	}
	return e
}

func DeepClone(original map[parser.MetricRequest][]*types.MetricData) map[parser.MetricRequest][]*types.MetricData {
	clone := map[parser.MetricRequest][]*types.MetricData{}
	for key, originalMetrics := range original {
		copiedMetrics := []*types.MetricData{}
		for _, originalMetric := range originalMetrics {
			copiedMetric := types.MetricData{
				FetchResponse: pb.FetchResponse{
					Name:      originalMetric.Name,
					StartTime: originalMetric.StartTime,
					StopTime:  originalMetric.StopTime,
					StepTime:  originalMetric.StepTime,
					Values:    make([]float64, len(originalMetric.Values)),
				},
			}

			copy(copiedMetric.Values, originalMetric.Values)
			copiedMetrics = append(copiedMetrics, &copiedMetric)
		}

		clone[key] = copiedMetrics
	}

	return clone
}

func compareFloat64(v1, v2 float64) bool {
	if math.IsNaN(v1) && math.IsNaN(v2) {
		return true
	}
	if math.IsInf(v1, 1) && math.IsInf(v2, 1) {
		return true
	}

	if math.IsInf(v1, 0) && math.IsInf(v2, 0) {
		return true
	}

	d := math.Abs(v1 - v2)
	return d < eps
}

func deepCompareFields(v1, v2 reflect.Value) bool {
	if !v1.CanInterface() {
		return true
	}
	t1 := v1.Type()
	if t1.Comparable() {
		if t1.Name() == "float64" {
			return compareFloat64(v1.Interface().(float64), v2.Interface().(float64))
		}
		if t1.Name() == "float32" {
			v1f64 := float64(v1.Interface().(float32))
			v2f64 := float64(v2.Interface().(float32))
			return compareFloat64(v1f64, v2f64)
		}
		return reflect.DeepEqual(v1.Interface(), v2.Interface())
	} else {
		switch v1.Kind() {
		case reflect.Struct:
			if v1.NumField() == 0 {
				// We don't know how to compare that
				return false
			}
			for i := 0; i < v1.NumField(); i++ {
				r := deepCompareFields(v1.Field(i), v2.Field(i))
				if !r {
					return r
				}
			}
		case reflect.Slice, reflect.Array:
			if v1.Len() != v2.Len() {
				return false
			}
			if v1.Len() == 0 {
				return true
			}
			if v1.Index(0).Kind() != v2.Index(0).Kind() {
				return false
			}
			for i := 0; i < v1.Len(); i++ {
				e1 := v1.Index(i)
				e2 := v2.Index(i)
				if !deepCompareFields(e1, e2) {
					return false
				}
			}
		case reflect.Func:
			return v1.Pointer() == v2.Pointer()
		default:
			fmt.Printf("unsupported v1.Kind=%v t1.Name=%v, t1.Value=%v\n\n", v1.Kind(), v1.Type().Name(), v1.String())
			return false
		}
	}
	return true
}
{'fraction_non_alphanumeric': 0.15668145956607496, 'fraction_numerical': 0.02638067061143984, 'mean_word_length': 3.766745005875441, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3048173', 'n_tokens_mistral': 3371, 'n_tokens_neox': 2980, 'n_words': 622}