<commit_before>package conf

import (
	"errors"
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/grafana/configparser"
)

func defaultAggregation() Aggregation {
	return Aggregation{
		Name:              "default",
		Pattern:           regexp.MustCompile(".*"),
		XFilesFactor:      0.5,
		AggregationMethod: []Method{Avg},
	}
}

// Aggregations holds the aggregation definitions
type Aggregations struct {
	Data               []Aggregation
	DefaultAggregation Aggregation
}

type Aggregation struct {
	Name              string         // mandatory (I *think* carbon allows empty name, but the docs say you should provide one)
	Pattern           *regexp.Regexp // mandatory (I *think* carbon allows empty pattern, the docs say you should provide one. but an empty string is still a pattern)
	XFilesFactor      float64        // optional. defaults to 0.5
	AggregationMethod []Method       // optional. defaults to ['average']
}

// NewAggregations create instance of Aggregations
func NewAggregations() Aggregations {
	return Aggregations{
		Data:               make([]Aggregation, 0),
		DefaultAggregation: defaultAggregation(),
	}
}

// ReadAggregations returns the defined aggregations from a storage-aggregation.conf file
// and adds the default
func ReadAggregations(file string) (Aggregations, error) {
	config, err := configparser.ReadFile(file)
	if err != nil {
		return Aggregations{}, err
	}
	_, sections, err := config.AllSections()
	if err != nil {
		return Aggregations{}, err
	}

	result := NewAggregations()

	for _, s := range sections {
		item := defaultAggregation()
		item.Name = s.Name()
		if item.Name == "" {
			return Aggregations{}, errors.New("encountered a storage-aggregation.conf section name with empty name")
		}

		if !s.Exists("pattern") {
			return Aggregations{}, fmt.Errorf("[%s]: missing pattern", item.Name)
		}

		item.Pattern, err = regexp.Compile(s.ValueOf("pattern"))
		if err != nil {
			return Aggregations{}, fmt.Errorf("[%s]: failed to parse pattern %q: %s", item.Name, s.ValueOf("pattern"), err.Error())
		}

		if s.Exists("xFilesFactor") {
			item.XFilesFactor, err = strconv.ParseFloat(s.ValueOf("xFilesFactor"), 64)
			if err != nil {
				return Aggregations{}, fmt.Errorf("[%s]: failed to parse xFilesFactor %q: %s", item.Name, s.ValueOf("xFilesFactor"), err.Error())
			}
		}

		if s.Exists("aggregationMethod") {
			aggregationMethodStr := s.ValueOf("aggregationMethod")
			methodStrs := strings.Split(aggregationMethodStr, ",")
			item.AggregationMethod = []Method{}
			for _, methodStr := range methodStrs {
				agg, err := NewMethod(methodStr)
				if err != nil {
					return result, fmt.Errorf("[%s]: %s", item.Name, err.Error())
				}
				item.AggregationMethod = append(item.AggregationMethod, agg)
			}
		}

		result.Data = append(result.Data, item)
	}<commit_msg>strip susppected comments from pattern, xFF and method directives<commit_after>package conf

import (
	"errors"
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/grafana/configparser"
)

func defaultAggregation() Aggregation {
	return Aggregation{
		Name:              "default",
		Pattern:           regexp.MustCompile(".*"),
		XFilesFactor:      0.5,
		AggregationMethod: []Method{Avg},
	}
}

// Aggregations holds the aggregation definitions
type Aggregations struct {
	Data               []Aggregation
	DefaultAggregation Aggregation
}

type Aggregation struct {
	Name              string         // mandatory (I *think* carbon allows empty name, but the docs say you should provide one)
	Pattern           *regexp.Regexp // mandatory (I *think* carbon allows empty pattern, the docs say you should provide one. but an empty string is still a pattern)
	XFilesFactor      float64        // optional. defaults to 0.5
	AggregationMethod []Method       // optional. defaults to ['average']
}

// NewAggregations create instance of Aggregations
func NewAggregations() Aggregations {
	return Aggregations{
		Data:               make([]Aggregation, 0),
		DefaultAggregation: defaultAggregation(),
	}
}

// ReadAggregations returns the defined aggregations from a storage-aggregation.conf file
// and adds the default
func ReadAggregations(file string) (Aggregations, error) {
	config, err := configparser.ReadFile(file)
	if err != nil {
		return Aggregations{}, err
	}
	_, sections, err := config.AllSections()
	if err != nil {
		return Aggregations{}, err
	}

	result := NewAggregations()

	for _, s := range sections {
		item := defaultAggregation()
		item.Name = s.Name()
		if item.Name == "" {
			return Aggregations{}, errors.New("encountered a storage-aggregation.conf section name with empty name")
		}

		if !s.Exists("pattern") {
			return Aggregations{}, fmt.Errorf("[%s]: missing pattern", item.Name)
		}

		// people may want to use # and ; in general as part of the metric name and regex (not a good idea but that's up to them)
		// but seems safe to assume that ' ;' and ' #' initiate a comment
		patt := s.ValueOfWithoutComments("pattern")
		item.Pattern, err = regexp.Compile(patt)
		if err != nil {
			return Aggregations{}, fmt.Errorf("[%s]: failed to parse pattern %q: %s", item.Name, patt, err.Error())
		}

		if s.Exists("xFilesFactor") {
			xff := s.ValueOfWithoutComments("xFilesFactor")
			item.XFilesFactor, err = strconv.ParseFloat(xff, 64)
			if err != nil {
				return Aggregations{}, fmt.Errorf("[%s]: failed to parse xFilesFactor %q: %s", item.Name, xff, err.Error())
			}
		}

		if s.Exists("aggregationMethod") {
			aggregationMethodStr := s.ValueOfWithoutComments("aggregationMethod")
			methodStrs := strings.Split(aggregationMethodStr, ",")
			item.AggregationMethod = []Method{}
			for _, methodStr := range methodStrs {
				agg, err := NewMethod(methodStr)
				if err != nil {
					return result, fmt.Errorf("[%s]: %s", item.Name, err.Error())
				}
				item.AggregationMethod = append(item.AggregationMethod, agg)
			}
		}

		result.Data = append(result.Data, item)
	}
{'fraction_non_alphanumeric': 0.12474713418745785, 'fraction_numerical': 0.0030343897505057315, 'mean_word_length': 3.5324675324675323, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3668426', 'n_tokens_mistral': 1961, 'n_tokens_neox': 1794, 'n_words': 527}