<commit_before>package main

import (
	"fmt"
	statsite "github.com/armon/go-metrics"
	"github.com/pshima/go-passenger-metrics/metrics"
	"log"
	"os"
	"time"
)

const (
	passengerPath string = "/usr/sbin/passenger-status"
	appname       string = "go-passenger-metrics"
)

var quiet bool = false

func main() {
	args := os.Args
	if len(args) > 1 {
		if args[1] == "-quiet" {
			quiet = true
		}
	}

	statsite_host := os.Getenv("GRAPHITE_HOST")
	statsite_port := os.Getenv("GRAPHITE_PORT")
	if statsite_host == "" {
		log.Fatal("GRAPHITE_HOST is empty, exiting")
	}
	if statsite_port == "" {
		log.Fatal("GRAPHITE_PORT is empty, exiting")
	}
	statsite_addr := fmt.Sprintf("%s:%s", statsite_host, statsite_port)
	log.Printf("%s Starting metrics output to %s", appname, statsite_addr)
	sink, err := statsite.NewStatsiteSink(statsite_addr)
	if err != nil {
		log.Fatalf("Error connecting", err)
	}
	if _, err := statsite.NewGlobal(statsite.DefaultConfig(appname), sink); err != nil {
		log.Fatalf("Error starting metrics layer")
	}
	ticker := time.NewTicker(10 * time.Second)
	for range ticker.C {
		run()
	}

}

func run() {
	p := &passengermetrics.PassengerCollection{}
	p.PassengerPath = passengerPath
	if err := p.RunPassengerStatus(); err != nil {
		log.Fatalf("%s Error running passenger status: %v", appname, err)
	}
	if err := p.ParseRawOutput(); err != nil {
		log.Fatalf("%s %v", appname, err)
	}

	if quiet != true {
		log.Printf("%s Queue Depth: %v", appname, p.ParsedOutput.QueueLength)
	}

	statsite.SetGauge([]string{"passenger-queue-depth"}, float32(p.ParsedOutput.QueueLength))
}
<commit_msg>Track the length of the raw output<commit_after>package main

import (
	"fmt"
	statsite "github.com/armon/go-metrics"
	"github.com/pshima/go-passenger-metrics/metrics"
	"log"
	"os"
	"time"
)

const (
	passengerPath string = "/usr/sbin/passenger-status"
	appname       string = "go-passenger-metrics"
)

var quiet bool = false

func main() {
	args := os.Args
	if len(args) > 1 {
		if args[1] == "-quiet" {
			quiet = true
		}
	}

	statsite_host := os.Getenv("GRAPHITE_HOST")
	statsite_port := os.Getenv("GRAPHITE_PORT")
	if statsite_host == "" {
		log.Fatal("GRAPHITE_HOST is empty, exiting")
	}
	if statsite_port == "" {
		log.Fatal("GRAPHITE_PORT is empty, exiting")
	}
	statsite_addr := fmt.Sprintf("%s:%s", statsite_host, statsite_port)
	log.Printf("%s Starting metrics output to %s", appname, statsite_addr)
	sink, err := statsite.NewStatsiteSink(statsite_addr)
	if err != nil {
		log.Fatalf("Error connecting", err)
	}
	if _, err := statsite.NewGlobal(statsite.DefaultConfig(appname), sink); err != nil {
		log.Fatalf("Error starting metrics layer")
	}
	ticker := time.NewTicker(10 * time.Second)
	for range ticker.C {
		run()
	}

}

func run() {
	p := &passengermetrics.PassengerCollection{}
	p.PassengerPath = passengerPath
	if err := p.RunPassengerStatus(); err != nil {
		log.Fatalf("%s Error running passenger status: %v", appname, err)
	}
	if err := p.ParseRawOutput(); err != nil {
		log.Fatalf("%s %v", appname, err)
	}

	if quiet != true {
		log.Printf("%s Queue Depth: %v", appname, p.ParsedOutput.QueueLength)
	}

	statsite.SetGauge([]string{"passenger-queue-depth"}, float32(p.ParsedOutput.QueueLength))
	statsite.SetGauge([]string{"passenger-stats-length"}, float32(len(p.RawOutput)))
}

{'fraction_non_alphanumeric': 0.15875415784699123, 'fraction_numerical': 0.0042334442092530995, 'mean_word_length': 4.352750809061488, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1898415', 'n_tokens_mistral': 1284, 'n_tokens_neox': 1240, 'n_words': 260}