<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/Dieterbe/statsd-go"
	"github.com/vimeo/statsdaemon"
	"github.com/vimeo/statsdaemon/timers"
	"io"
	"net"
	"os"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"
)

var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
var memprofile = flag.String("memprofile", "", "write memory profile to this file")

func main() {
	runtime.GOMAXPROCS(4)
	flag.Parse()
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}
	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			panic(err)
		}
		defer f.Close()
		defer pprof.WriteHeapProfile(f)
	}

	cl, err := statsd.NewClient(true, "localhost:8125", "statsd-tester")
	if nil != err {
		panic(err)
	}
	laddr, err := net.ResolveTCPAddr("tcp", "localhost:2003")
	if nil != err {
		panic(err)
	}
	w := NewWatcher(laddr)
	go w.Run()
	pct := timers.Percentiles{}
	daemon := statsdaemon.New("test", ":8125", ":8126", ":2003", "rates.", "timers.", "gauges.", pct, 10, 1000, 1000, nil, false)
	tick := time.Tick(time.Duration(1) * time.Second)
	go func() {
		for range tick {
			// send 1M packets per second in theory. in practice this takes more than a second
			msg := []byte("test.counter:1|c")
			for i := 0; i < 1000000; i++ {
				//cl.Increment("test-counter")
				cl.SendRaw(msg)
			}
		}
	}()

	daemon.Run()
}

type watcher struct {
	l      *net.TCPListener
	seen   int
	values chan string
}

func NewWatcher(laddr *net.TCPAddr) *watcher {
	l, err := net.ListenTCP("tcp", laddr)<commit_msg>better statsd client & metrics sending<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/alexcesaro/statsd"
	"github.com/vimeo/statsdaemon"
	"github.com/vimeo/statsdaemon/timers"
	"io"
	"net"
	"os"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"
)

var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
var memprofile = flag.String("memprofile", "", "write memory profile to this file")

func main() {
	runtime.GOMAXPROCS(4)
	flag.Parse()
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}
	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			panic(err)
		}
		defer f.Close()
		defer pprof.WriteHeapProfile(f)
	}

	laddr, err := net.ResolveTCPAddr("tcp", "localhost:2003")
	if nil != err {
		panic(err)
	}
	w := NewWatcher(laddr)
	go w.Run()
	pct := timers.Percentiles{}
	daemon := statsdaemon.New("test", ":8125", ":8126", ":2003", "rates.", "timers.", "gauges.", pct, 10, 1000, 1000, nil, false)
	go daemon.Run()
	//daemon must be running otherwise client complains
	time.Sleep(200 * time.Millisecond)
	cl, err := statsd.New(":8125", statsd.WithPrefix("statsd-tester"), statsd.WithFlushPeriod(10*time.Millisecond))
	if nil != err {
		panic(err)
	}

	tick := time.Tick(time.Duration(10) * time.Millisecond)
	go func() {
		for range tick {
			for i := 0; i < 10000; i++ {
				cl.Count("test-counter", 1, 1)

			}
		}
	}()
	time.Sleep(10 * 11 * time.Second)
	panic("timeout: watcher did not get all updates from statsdaemon")

}

type watcher struct {
	l      *net.TCPListener
	seen   int
	values chan string
}

func NewWatcher(laddr *net.TCPAddr) *watcher {
	l, err := net.ListenTCP("tcp", laddr)
{'fraction_non_alphanumeric': 0.17227259432582626, 'fraction_numerical': 0.02690845276396607, 'mean_word_length': 3.776536312849162, 'pattern_counts': {'":': 7, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '863558', 'n_tokens_mistral': 1489, 'n_tokens_neox': 1353, 'n_words': 311}