<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/jcoene/gologger"
	"io/ioutil"
	"math"
	"net"
	"net/http"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

type Packet struct {
	Bucket   string
	Value    string
	Modifier string
	Sampling float32
}

var log *logger.Logger
var sanitizeRegexp = regexp.MustCompile("[^a-zA-Z0-9\\-_\\.:\\|@]")
var packetRegexp = regexp.MustCompile("([a-zA-Z0-9_\\.]+):(\\-?[0-9\\.]+)\\|(c|ms|g)(\\|@([0-9\\.]+))?")

var (
	serviceAddress = flag.String("address", "0.0.0.0:8125", "udp listen address")
	libratoUser    = flag.String("user", "", "librato api username (LIBRATO_USER)")
	libratoToken   = flag.String("token", "", "librato api token (LIBRATO_TOKEN)")
	libratoSource  = flag.String("source", "", "librato api source (LIBRATO_SOURCE)")
	flushInterval  = flag.Int64("flush", 60, "interval at which data is sent to librato (in seconds)")
	percentiles    = flag.String("percentiles", "", "comma separated list of percentiles to calculate for timers (eg. \"95,99.5\")")
	debug          = flag.Bool("debug", false, "enable logging of inputs and submissions")
)

var (
	In       = make(chan Packet, 10000)
	counters = make(map[string]int64)
	timers   = make(map[string][]float64)
	gauges   = make(map[string]float64)
)

type Measurement struct {
	Counters []Counter     `json:"counters"`
	Gauges   []interface{} `json:"gauges"`
	Source   *string       `json:"source,omitempty"`
}

func (m *Measurement) Count() int {
	return (len(m.Counters) + len(m.Gauges))
}

type Counter struct {
	Name  string `json:"name"`
	Value int64  `json:"value"`
}

type SimpleGauge struct {
	Name  string  `json:"name"`
	Value float64 `json:"value"`
}

type ComplexGauge struct {
	Name       string  `json:"name"`
	Count      int     `json:"count"`
	Sum        float64 `json:"sum"`
	Min        float64 `json:"min"`
	Max        float64 `json:"max"`
	SumSquares float64 `json:"sum_squares"`
	Median     float64 `json:"median"`
}

func monitor() {
	t := time.NewTicker(time.Duration(*flushInterval) * time.Second)

	for {
		select {
		case <-t.C:
			if err := submit(); err != nil {
				log.Error("submit: %s", err)
			}
		case s := <-In:
			if s.Modifier == "ms" {
				_, ok := timers[s.Bucket]
				if !ok {
					var t []float64
					timers[s.Bucket] = t
				}
				floatValue, _ := strconv.ParseFloat(s.Value, 64)
				timers[s.Bucket] = append(timers[s.Bucket], floatValue)
			} else if s.Modifier == "g" {
				_, ok := gauges[s.Bucket]
				if !ok {
					gauges[s.Bucket] = float64(0)
				}
				floatValue, _ := strconv.ParseFloat(s.Value, 64)
				gauges[s.Bucket] = floatValue
			} else {
				_, ok := counters[s.Bucket]
				if !ok {
					counters[s.Bucket] = 0
				}
				floatValue, _ := strconv.ParseFloat(s.Value, 32)
				counters[s.Bucket] += int64(float32(floatValue) * (1 / s.Sampling))
			}
		}
	}
}

func submit() (err error) {
	m := new(Measurement)
	if *libratoSource != "" {
		m.Source = libratoSource
	}
	m.Counters = make([]Counter, 0)
	m.Gauges = make([]interface{}, 0)

	for k, v := range counters {
		c := new(Counter)
		c.Name = k
		c.Value = v
		m.Counters = append(m.Counters, *c)
	}

	for k, v := range gauges {
		g := new(SimpleGauge)
		g.Name = k
		g.Value = v
		m.Gauges = append(m.Gauges, *g)
	}

	for k, t := range timers {
		g := gaugePercentile(k, t, 100.0, "")
		m.Gauges = append(m.Gauges, *g)

		pcts := strings.Split(*percentiles, ",")
		for _, pct := range pcts {
			pctf, err := strconv.ParseFloat(pct, 64)
			if err != nil {
				log.Warn("error parsing %s as float: %s", pct, m.Count(), err)
				continue
			}

			g = gaugePercentile(k, t, pctf, "pct"+pct)
			m.Gauges = append(m.Gauges, *g)
		}
	}<commit_msg>Handle no percentiles value more carefully<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/jcoene/gologger"
	"io/ioutil"
	"math"
	"net"
	"net/http"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

type Packet struct {
	Bucket   string
	Value    string
	Modifier string
	Sampling float32
}

var log *logger.Logger
var sanitizeRegexp = regexp.MustCompile("[^a-zA-Z0-9\\-_\\.:\\|@]")
var packetRegexp = regexp.MustCompile("([a-zA-Z0-9_\\.]+):(\\-?[0-9\\.]+)\\|(c|ms|g)(\\|@([0-9\\.]+))?")

var (
	serviceAddress = flag.String("address", "0.0.0.0:8125", "udp listen address")
	libratoUser    = flag.String("user", "", "librato api username (LIBRATO_USER)")
	libratoToken   = flag.String("token", "", "librato api token (LIBRATO_TOKEN)")
	libratoSource  = flag.String("source", "", "librato api source (LIBRATO_SOURCE)")
	flushInterval  = flag.Int64("flush", 60, "interval at which data is sent to librato (in seconds)")
	percentiles    = flag.String("percentiles", "", "comma separated list of percentiles to calculate for timers (eg. \"95,99.5\")")
	debug          = flag.Bool("debug", false, "enable logging of inputs and submissions")
)

var (
	In       = make(chan Packet, 10000)
	counters = make(map[string]int64)
	timers   = make(map[string][]float64)
	gauges   = make(map[string]float64)
)

type Measurement struct {
	Counters []Counter     `json:"counters"`
	Gauges   []interface{} `json:"gauges"`
	Source   *string       `json:"source,omitempty"`
}

func (m *Measurement) Count() int {
	return (len(m.Counters) + len(m.Gauges))
}

type Counter struct {
	Name  string `json:"name"`
	Value int64  `json:"value"`
}

type SimpleGauge struct {
	Name  string  `json:"name"`
	Value float64 `json:"value"`
}

type ComplexGauge struct {
	Name       string  `json:"name"`
	Count      int     `json:"count"`
	Sum        float64 `json:"sum"`
	Min        float64 `json:"min"`
	Max        float64 `json:"max"`
	SumSquares float64 `json:"sum_squares"`
	Median     float64 `json:"median"`
}

func monitor() {
	t := time.NewTicker(time.Duration(*flushInterval) * time.Second)

	for {
		select {
		case <-t.C:
			if err := submit(); err != nil {
				log.Error("submit: %s", err)
			}
		case s := <-In:
			if s.Modifier == "ms" {
				_, ok := timers[s.Bucket]
				if !ok {
					var t []float64
					timers[s.Bucket] = t
				}
				floatValue, _ := strconv.ParseFloat(s.Value, 64)
				timers[s.Bucket] = append(timers[s.Bucket], floatValue)
			} else if s.Modifier == "g" {
				_, ok := gauges[s.Bucket]
				if !ok {
					gauges[s.Bucket] = float64(0)
				}
				floatValue, _ := strconv.ParseFloat(s.Value, 64)
				gauges[s.Bucket] = floatValue
			} else {
				_, ok := counters[s.Bucket]
				if !ok {
					counters[s.Bucket] = 0
				}
				floatValue, _ := strconv.ParseFloat(s.Value, 32)
				counters[s.Bucket] += int64(float32(floatValue) * (1 / s.Sampling))
			}
		}
	}
}

func submit() (err error) {
	m := new(Measurement)
	if *libratoSource != "" {
		m.Source = libratoSource
	}
	m.Counters = make([]Counter, 0)
	m.Gauges = make([]interface{}, 0)

	for k, v := range counters {
		c := new(Counter)
		c.Name = k
		c.Value = v
		m.Counters = append(m.Counters, *c)
	}

	for k, v := range gauges {
		g := new(SimpleGauge)
		g.Name = k
		g.Value = v
		m.Gauges = append(m.Gauges, *g)
	}

	for k, t := range timers {
		g := gaugePercentile(k, t, 100.0, "")
		m.Gauges = append(m.Gauges, *g)

		if *percentiles != "" {
			pcts := strings.Split(*percentiles, ",")
			log.Info("pctiles: %+v", pcts)
			for _, pct := range pcts {
				pctf, err := strconv.ParseFloat(pct, 64)
				if err != nil {
					log.Warn("error parsing '%s' as float: %s", pct, err)
					continue
				}

				g = gaugePercentile(k, t, pctf, "pct"+pct)
				m.Gauges = append(m.Gauges, *g)
			}
		}
	}
{'fraction_non_alphanumeric': 0.17729930887825623, 'fraction_numerical': 0.02046783625730994, 'mean_word_length': 3.17823431426985, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '687529', 'n_tokens_mistral': 3326, 'n_tokens_neox': 2958, 'n_words': 661}