<commit_before>package statsd

import (
	"fmt"
	"math"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/jtblin/gostatsd/backend"
	"github.com/jtblin/gostatsd/types"

	log "github.com/Sirupsen/logrus"
)

// metricAggregatorStats is a bookkeeping structure for statistics about a MetricAggregator
type metricAggregatorStats struct {
	BadLines       int64
	LastMessage    time.Time
	LastFlush      time.Time
	LastFlushError time.Time
	NumStats       int
	ProcessingTime time.Duration
}

// MetricAggregator is an object that aggregates statsd metrics.
// The function NewMetricAggregator should be used to create the objects.
//
// Incoming metrics should be sent to the MetricChan channel.
type MetricAggregator struct {
	sync.Mutex
	ExpiryInterval    time.Duration     // How often to expire metrics
	FlushInterval     time.Duration     // How often to flush metrics to the sender
	MaxWorkers        int               // Number of workers to metrics queue
	MetricQueue       chan types.Metric // Queue on which metrics are received
	PercentThresholds []float64
	Senders           []backend.MetricSender // The sender to which metrics are flushed
	Stats             metricAggregatorStats
	Tags              types.Tags // Tags to add to all metrics
	types.MetricMap
}

// NewMetricAggregator creates a new MetricAggregator object
func NewMetricAggregator(senders []backend.MetricSender, percentThresholds []float64, flushInterval time.Duration, expiryInterval time.Duration, maxWorkers int, tags []string) *MetricAggregator {
	a := MetricAggregator{}
	a.FlushInterval = flushInterval
	a.ExpiryInterval = expiryInterval
	a.Senders = senders
	a.MetricQueue = make(chan types.Metric, maxQueueSize * 10) // we are going to receive more metrics than messages
	a.MaxWorkers = maxWorkers
	a.PercentThresholds = percentThresholds
	a.Counters = types.Counters{}
	a.Timers = types.Timers{}
	a.Gauges = types.Gauges{}
	a.Sets = types.Sets{}
	a.Tags = tags
	return &a<commit_msg>Fix comment<commit_after>package statsd

import (
	"fmt"
	"math"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/jtblin/gostatsd/backend"
	"github.com/jtblin/gostatsd/types"

	log "github.com/Sirupsen/logrus"
)

// metricAggregatorStats is a bookkeeping structure for statistics about a MetricAggregator
type metricAggregatorStats struct {
	BadLines       int64
	LastMessage    time.Time
	LastFlush      time.Time
	LastFlushError time.Time
	NumStats       int
	ProcessingTime time.Duration
}

// MetricAggregator is an object that aggregates statsd metrics.
// The function NewMetricAggregator should be used to create the objects.
//
// Incoming metrics should be sent to the MetricQueue channel.
type MetricAggregator struct {
	sync.Mutex
	ExpiryInterval    time.Duration     // How often to expire metrics
	FlushInterval     time.Duration     // How often to flush metrics to the sender
	MaxWorkers        int               // Number of workers to metrics queue
	MetricQueue       chan types.Metric // Queue on which metrics are received
	PercentThresholds []float64
	Senders           []backend.MetricSender // The sender to which metrics are flushed
	Stats             metricAggregatorStats
	Tags              types.Tags // Tags to add to all metrics
	types.MetricMap
}

// NewMetricAggregator creates a new MetricAggregator object
func NewMetricAggregator(senders []backend.MetricSender, percentThresholds []float64, flushInterval time.Duration, expiryInterval time.Duration, maxWorkers int, tags []string) *MetricAggregator {
	a := MetricAggregator{}
	a.FlushInterval = flushInterval
	a.ExpiryInterval = expiryInterval
	a.Senders = senders
	a.MetricQueue = make(chan types.Metric, maxQueueSize*10) // we are going to receive more metrics than messages
	a.MaxWorkers = maxWorkers
	a.PercentThresholds = percentThresholds
	a.Counters = types.Counters{}
	a.Timers = types.Timers{}
	a.Gauges = types.Gauges{}
	a.Sets = types.Sets{}
	a.Tags = tags
	return &a
{'fraction_non_alphanumeric': 0.07337579617834394, 'fraction_numerical': 0.004076433121019108, 'mean_word_length': 3.607981220657277, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4399990', 'n_tokens_mistral': 1190, 'n_tokens_neox': 1181, 'n_words': 365}