<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"regexp"
	"strings"
	"time"

	"gopkg.in/buildkite/go-buildkite.v2/buildkite"
)

const recordsPerPage = 100

// Version is passed in via ldflags
var Version string

var queuePattern *regexp.Regexp

func init() {
	queuePattern = regexp.MustCompile(`(?i)^queue=(.+?)$`)
}

func main() {
	var (
		accessToken = flag.String("token", "", "A Buildkite API Access Token")
		orgSlug     = flag.String("org", "", "A Buildkite Organization Slug")
		interval    = flag.Duration("interval", 0, "Update metrics every interval, rather than once")
		history     = flag.Duration("history", time.Hour*24, "Historical data to use for finished builds")
		debug       = flag.Bool("debug", false, "Show API debugging output")
		version     = flag.Bool("version", false, "Show the version")

		// filters
		queues = flag.String("queue", "", "Filter by list of queues, comma delimited")
	)

	flag.Parse()

	if *version {
		fmt.Printf("buildkite-metrics %s\n", Version)
		os.Exit(0)
	}

	if *accessToken == "" {
		log.Fatal("Must provide a value for -token")
	}

	if *orgSlug == "" {
		log.Fatal("Must provide a value for -org")
	}

	config, err := buildkite.NewTokenConfig(*accessToken, false)
	if err != nil {
		log.Fatalf("client config failed: %s", err)
	}

	client := buildkite.NewClient(config.Client())
	buildkite.SetHttpDebug(*debug)

	f := func() error {
		t := time.Now()

		res, err := collectResults(client, collectOpts{
			OrgSlug:    *orgSlug,
			Historical: *history,
			Queues:     strings.Split(*queues, ","),
		})
		if err != nil {
			return err
		}

		dumpResults(res)

		err = cloudwatchSend(res)
		if err != nil {
			return err
		}

		log.Printf("Finished in %s", time.Now().Sub(t))
		return nil
	}

	if err := f(); err != nil {
		log.Fatal(err)
	}

	if *interval > 0 {
		for _ = range time.NewTicker(*interval).C {
			if err := f(); err != nil {
				log.Println(err)
			}
		}
	}
}

type collectOpts struct {
	OrgSlug    string
	Historical time.Duration
	Queues     []string
}

func collectResults(client *buildkite.Client, opts collectOpts) (*result, error) {
	res := &result{
		totals:    newCounts(),
		queues:    map[string]counts{},
		pipelines: map[string]counts{},
	}

	if len(opts.Queues) == 0 {
		log.Println("Collecting historical metrics")
		if err := res.addHistoricalMetrics(client, opts); err != nil {
			return nil, err
		}
	}

	log.Println("Collecting running and scheduled build and job metrics")
	if err := res.addBuildAndJobMetrics(client, opts); err != nil {
		return nil, err
	}

	log.Println("Collecting agent metrics")
	if err := res.addAgentMetrics(client, opts); err != nil {
		return nil, err
	}

	if len(opts.Queues) == 0 {
		return res, nil
	}

	filteredRes := &result{
		queues: map[string]counts{},
	}

	for _, queue := range opts.Queues {
		filteredRes.queues[queue] = res.queues[queue]
	}

	return filteredRes, nil
}

func dumpResults(res *result) {
	for name, c := range res.totals {
		log.Printf("Buildkite > %s = %d", name, c)
	}

	for name, c := range res.queues {
		for k, v := range c {
			log.Printf("Buildkite > [queue = %s] > %s = %d", name, k, v)
		}
	}

	for name, c := range res.pipelines {
		for k, v := range c {
			log.Printf("Buildkite > [pipeline = %s] > %s = %d", name, k, v)
		}
	}
}

const (<commit_msg>Revert "Support multiple queues via --queue"<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"regexp"
	"time"

	"gopkg.in/buildkite/go-buildkite.v2/buildkite"
)

const recordsPerPage = 100

// Version is passed in via ldflags
var Version string

var queuePattern *regexp.Regexp

func init() {
	queuePattern = regexp.MustCompile(`(?i)^queue=(.+?)$`)
}

func main() {
	var (
		accessToken = flag.String("token", "", "A Buildkite API Access Token")
		orgSlug     = flag.String("org", "", "A Buildkite Organization Slug")
		interval    = flag.Duration("interval", 0, "Update metrics every interval, rather than once")
		history     = flag.Duration("history", time.Hour*24, "Historical data to use for finished builds")
		debug       = flag.Bool("debug", false, "Show API debugging output")
		version     = flag.Bool("version", false, "Show the version")

		// filters
		queue = flag.String("queue", "", "Only include a specific queue")
	)

	flag.Parse()

	if *version {
		fmt.Printf("buildkite-metrics %s\n", Version)
		os.Exit(0)
	}

	if *accessToken == "" {
		log.Fatal("Must provide a value for -token")
	}

	if *orgSlug == "" {
		log.Fatal("Must provide a value for -org")
	}

	config, err := buildkite.NewTokenConfig(*accessToken, false)
	if err != nil {
		log.Fatalf("client config failed: %s", err)
	}

	client := buildkite.NewClient(config.Client())
	buildkite.SetHttpDebug(*debug)

	f := func() error {
		t := time.Now()

		res, err := collectResults(client, collectOpts{
			OrgSlug:    *orgSlug,
			Historical: *history,
			Queue:      *queue,
		})
		if err != nil {
			return err
		}

		dumpResults(res)

		err = cloudwatchSend(res)
		if err != nil {
			return err
		}

		log.Printf("Finished in %s", time.Now().Sub(t))
		return nil
	}

	if err := f(); err != nil {
		log.Fatal(err)
	}

	if *interval > 0 {
		for _ = range time.NewTicker(*interval).C {
			if err := f(); err != nil {
				log.Println(err)
			}
		}
	}
}

type collectOpts struct {
	OrgSlug    string
	Historical time.Duration
	Queue      string
}

func collectResults(client *buildkite.Client, opts collectOpts) (*result, error) {
	res := &result{
		totals:    newCounts(),
		queues:    map[string]counts{},
		pipelines: map[string]counts{},
	}

	if opts.Queue == "" {
		log.Println("Collecting historical metrics")
		if err := res.addHistoricalMetrics(client, opts); err != nil {
			return nil, err
		}
	}

	log.Println("Collecting running and scheduled build and job metrics")
	if err := res.addBuildAndJobMetrics(client, opts); err != nil {
		return nil, err
	}

	log.Println("Collecting agent metrics")
	if err := res.addAgentMetrics(client, opts); err != nil {
		return nil, err
	}

	if opts.Queue != "" {
		if c, ok := res.queues[opts.Queue]; ok {
			return &result{
				queues: map[string]counts{
					opts.Queue: c,
				},
			}, nil
		}
		return &result{}, nil
	}

	return res, nil
}

func dumpResults(res *result) {
	for name, c := range res.totals {
		log.Printf("Buildkite > %s = %d", name, c)
	}

	for name, c := range res.queues {
		for k, v := range c {
			log.Printf("Buildkite > [queue = %s] > %s = %d", name, k, v)
		}
	}

	for name, c := range res.pipelines {
		for k, v := range c {
			log.Printf("Buildkite > [pipeline = %s] > %s = %d", name, k, v)
		}
	}
}

const (
{'fraction_non_alphanumeric': 0.15713635674199908, 'fraction_numerical': 0.003033520400424693, 'mean_word_length': 3.329612606697308, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1573314', 'n_tokens_mistral': 2610, 'n_tokens_neox': 2386, 'n_words': 634}