<commit_before>package expr

import (
	"bytes"
	"errors"
	"sort"

	"github.com/grafana/metrictank/api/models"
	"github.com/raintank/schema"
)

type FuncGroupByTags struct {
	in         GraphiteFunc
	aggregator string
	tags       []string
}

func NewGroupByTags() GraphiteFunc {
	return &FuncGroupByTags{}
}

func (s *FuncGroupByTags) Signature() ([]Arg, []Arg) {
	return []Arg{
		ArgSeriesList{val: &s.in},
		ArgString{val: &s.aggregator, validator: []Validator{IsAggFunc}},
		ArgStrings{val: &s.tags},
	}, []Arg{ArgSeries{}}
}

func (s *FuncGroupByTags) Context(context Context) Context {
	return context
}

func (s *FuncGroupByTags) Exec(cache map[Req][]models.Series) ([]models.Series, error) {
	series, err := s.in.Exec(cache)
	if err != nil {
		return nil, err
	}

	if len(series) == 0 {
		return series, nil
	}

	if len(s.tags) == 0 {
		return nil, errors.New("No tags specified")
	}

	groups := make(map[string][]models.Series)
	useName := false

	groupTags := s.tags
	for i, tag := range groupTags {
		if tag == "name" {
			// We handle name explicitly, remove it from tags
			useName = true
			groupTags = append(groupTags[:i], groupTags[i+1:]...)
			break
		}
	}

	nameReplace := ""
	if !useName {
		// if all series have the same name, name becomes one of our tags
		for _, serie := range series {
			thisName := serie.Tags["name"]
			if nameReplace == "" {
				nameReplace = thisName
			} else if nameReplace != thisName {
				nameReplace = s.aggregator
				break
			}
		}
	}

	// Tags need to be sorted
	sort.Strings(groupTags)

	// First pass - group our series together by key
	var buffer bytes.Buffer
	for _, serie := range series {
		buffer.Reset()

		if useName {
			buffer.WriteString(serie.Tags["name"])
		} else {
			buffer.WriteString(nameReplace)
		}

		for _, goal := range groupTags {
			buffer.WriteRune(';')
			buffer.WriteString(goal)
			buffer.WriteRune('=')

			tagVal, ok := serie.Tags[goal]
			if ok {
				buffer.WriteString(tagVal)
			}
		}

		key := buffer.String()

		groups[key] = append(groups[key], serie)
	}

	output := make([]models.Series, 0, len(groups))
	aggFunc := getCrossSeriesAggFunc(s.aggregator)

	// Now, for each key perform the requested aggregation
	cons, queryCons := summarizeCons(series)

	for name, groupSeries := range groups {
		newSeries := models.Series{
			Target:       name,
			QueryPatt:    name,
			Interval:     series[0].Interval,
			Consolidator: cons,
			QueryCons:    queryCons,
		}<commit_msg>Call summarizeCons for each group<commit_after>package expr

import (
	"bytes"
	"errors"
	"sort"

	"github.com/grafana/metrictank/api/models"
	"github.com/raintank/schema"
)

type FuncGroupByTags struct {
	in         GraphiteFunc
	aggregator string
	tags       []string
}

func NewGroupByTags() GraphiteFunc {
	return &FuncGroupByTags{}
}

func (s *FuncGroupByTags) Signature() ([]Arg, []Arg) {
	return []Arg{
		ArgSeriesList{val: &s.in},
		ArgString{val: &s.aggregator, validator: []Validator{IsAggFunc}},
		ArgStrings{val: &s.tags},
	}, []Arg{ArgSeries{}}
}

func (s *FuncGroupByTags) Context(context Context) Context {
	return context
}

func (s *FuncGroupByTags) Exec(cache map[Req][]models.Series) ([]models.Series, error) {
	series, err := s.in.Exec(cache)
	if err != nil {
		return nil, err
	}

	if len(series) == 0 {
		return series, nil
	}

	if len(s.tags) == 0 {
		return nil, errors.New("No tags specified")
	}

	groups := make(map[string][]models.Series)
	useName := false

	groupTags := s.tags
	for i, tag := range groupTags {
		if tag == "name" {
			// We handle name explicitly, remove it from tags
			useName = true
			groupTags = append(groupTags[:i], groupTags[i+1:]...)
			break
		}
	}

	nameReplace := ""
	if !useName {
		// if all series have the same name, name becomes one of our tags
		for _, serie := range series {
			thisName := serie.Tags["name"]
			if nameReplace == "" {
				nameReplace = thisName
			} else if nameReplace != thisName {
				nameReplace = s.aggregator
				break
			}
		}
	}

	// Tags need to be sorted
	sort.Strings(groupTags)

	// First pass - group our series together by key
	var buffer bytes.Buffer
	for _, serie := range series {
		buffer.Reset()

		if useName {
			buffer.WriteString(serie.Tags["name"])
		} else {
			buffer.WriteString(nameReplace)
		}

		for _, goal := range groupTags {
			buffer.WriteRune(';')
			buffer.WriteString(goal)
			buffer.WriteRune('=')

			tagVal, ok := serie.Tags[goal]
			if ok {
				buffer.WriteString(tagVal)
			}
		}

		key := buffer.String()

		groups[key] = append(groups[key], serie)
	}

	output := make([]models.Series, 0, len(groups))
	aggFunc := getCrossSeriesAggFunc(s.aggregator)

	// Now, for each key perform the requested aggregation
	for name, groupSeries := range groups {
		cons, queryCons := summarizeCons(groupSeries)

		newSeries := models.Series{
			Target:       name,
			QueryPatt:    name,
			Interval:     series[0].Interval,
			Consolidator: cons,
			QueryCons:    queryCons,
		}
{'fraction_non_alphanumeric': 0.1368038740920097, 'fraction_numerical': 0.0020177562550443904, 'mean_word_length': 3.4101423487544484, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '322676', 'n_tokens_mistral': 1903, 'n_tokens_neox': 1695, 'n_words': 438}