<commit_before>package vitals

import (
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"time"
)

var (
	nullLog = log.New(ioutil.Discard, "", 0)
)

func StartCPUProfile(file string) (deferFunc func(), err error) {
	if file == "" {
		return func() {}, nil
	}

	f, err := os.Create(file)
	if err != nil {
		return func() {}, err
	}
	if err = pprof.StartCPUProfile(f); err != nil {
		return func() {}, err
	}

	fn := func() {
		pprof.StopCPUProfile()
	}
	return fn, nil
}

func LogMemStats(cycle time.Duration, logger *log.Logger) {
	if cycle == 0 {
		return
	}

	if logger == nil {
		logger = nullLog
	}

	go func() {
		memStats := runtime.MemStats{}
		for {
			runtime.ReadMemStats(&memStats)
			logger.Printf(
				"%d, %d, %d, %d\n",
				memStats.HeapSys,
				memStats.HeapAlloc,
				memStats.HeapIdle,
				memStats.HeapReleased,
			)
			time.Sleep(cycle)
		}
	}()
}

func WriteHeapProfile(file string) error {
	if file == "" {
		return nil
	}

	f, err := os.Create(file)
	if err != nil {<commit_msg>Tentatively change memstats output for clarity.<commit_after>package vitals

import (
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"time"
)

var (
	nullLog = log.New(ioutil.Discard, "", 0)
)

func StartCPUProfile(file string) (deferFunc func(), err error) {
	if file == "" {
		return func() {}, nil
	}

	f, err := os.Create(file)
	if err != nil {
		return func() {}, err
	}
	if err = pprof.StartCPUProfile(f); err != nil {
		return func() {}, err
	}

	fn := func() {
		pprof.StopCPUProfile()
	}
	return fn, nil
}

func LogMemStats(cycle time.Duration, logger *log.Logger) {
	if cycle == 0 {
		return
	}

	if logger == nil {
		logger = nullLog
	}

	go func() {
		memStats := runtime.MemStats{}
		for {
			runtime.ReadMemStats(&memStats)
			logger.Printf(
				"CurAlloc(kB): %d, FromSys(kB): %d, CurMalloc: %d\n",
				memStats.Alloc/1000,
				memStats.Sys/1000,
				(memStats.Mallocs - memStats.Frees),
			)
			time.Sleep(cycle)
		}
	}()
}

func WriteHeapProfile(file string) error {
	if file == "" {
		return nil
	}

	f, err := os.Create(file)
	if err != nil {
{'fraction_non_alphanumeric': 0.16466056812710642, 'fraction_numerical': 0.005777563793933558, 'mean_word_length': 3.114851485148515, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5024844', 'n_tokens_mistral': 910, 'n_tokens_neox': 831, 'n_words': 191}