<commit_before>package proc

import (
	"fmt"
	"os"
	"time"
)

type (
	Counts struct {
		Cpu        float64
		ReadBytes  uint64
		WriteBytes uint64
	}

	Memory struct {
		Resident uint64
		Virtual  uint64
	}

	Filedesc struct {
		Open  int64 // -1 if we don't know
		Limit uint64
	}

	// Tracker tracks processes and records metrics.
	Tracker struct {
		// Tracked holds the processes are being monitored.  Processes
		// may be blacklisted such that they no longer get tracked by
		// setting their value in the Tracked map to nil.
		Tracked map[ProcId]*TrackedProc
		// ProcIds is a map from pid to ProcId.  This is a convenience
		// to allow finding the Tracked entry of a parent process.
		ProcIds map[int]ProcId
	}

	// TrackedProc accumulates metrics for a process, as well as
	// remembering an optional GroupName tag associated with it.
	TrackedProc struct {
		// lastUpdate is used internally during the update cycle to find which procs have exited
		lastUpdate time.Time
		// info is the most recently obtained info for this proc
		info ProcInfo
		// accum is the total CPU and IO accrued since we started tracking this proc
		accum Counts
		// lastaccum is the CPU and IO accrued in the last Update()
		lastaccum Counts
		// GroupName is an optional tag for this proc.
		GroupName string
	}

	trackedStats struct {
		aggregate, latest Counts
		Memory
		Filedesc
		start time.Time
	}

	collectErrors struct {
		// readErrors is incremented every time GetMetrics() returns an error.
		Read int
		// permissionErrors is incremented every time we're unable to collect
		// some metrics (e.g. I/O) for a tracked proc.
		Permission int
	}
)

func (tp *TrackedProc) GetName() string {
	return tp.info.Name
}

func (tp *TrackedProc) GetCmdLine() []string {
	return tp.info.Cmdline
}

func (tp *TrackedProc) GetStats() trackedStats {
	mem := Memory{Resident: tp.info.ResidentBytes, Virtual: tp.info.VirtualBytes}
	fd := Filedesc{Open: tp.info.OpenFDs, Limit: tp.info.MaxFDs}
	return trackedStats{
		aggregate: tp.accum,
		latest:    tp.lastaccum,
		Memory:    mem,
		Filedesc:  fd,
		start:     tp.info.StartTime,
	}
}

func NewTracker() *Tracker {
	return &Tracker{Tracked: make(map[ProcId]*TrackedProc), ProcIds: make(map[int]ProcId)}
}

func (t *Tracker) Track(groupName string, idinfo ProcIdInfo) {
	info := ProcInfo{idinfo.ProcStatic, idinfo.ProcMetrics}
	t.Tracked[idinfo.ProcId] = &TrackedProc{GroupName: groupName, info: info}
}

func (t *Tracker) Ignore(id ProcId) {
	t.Tracked[id] = nil
}

// Scan procs and update metrics for those which are tracked.  Processes that have gone
// away get removed from the Tracked map.  New processes are returned, along with the count
// of permission errors.
func (t *Tracker) Update(procs ProcIter) ([]ProcIdInfo, collectErrors, error) {
	now := time.Now()
	var newProcs []ProcIdInfo
	var colErrs collectErrors

	for procs.Next() {
		procId, err := procs.GetProcId()
		if err != nil {
			continue
		}

		last, known := t.Tracked[procId]

		// Are we ignoring this proc?
		if known && last == nil {
			continue
		}

		metrics, err := procs.GetMetrics()
		if err != nil {
			// This usually happens due to the proc having exited, i.e.
			// we lost the race.  We don't count that as an error.
			if err != ErrProcNotExist {
				fmt.Fprintln(os.Stderr, "error reading metrics for ", procId, err)
				colErrs.Read++
			}
			continue
		}

		if known {
			// newcounts: resource consumption since last cycle
			newcounts := Counts{Cpu: metrics.CpuTime - last.info.CpuTime}
			// Counts are also used in Grouper, to track aggregate usage
			// across groups.  It would be nice to expose that we weren't
			// able to get some metrics (e.g. due to permissions) but not nice<commit_msg>Remove debug log message committed accidentally.<commit_after>package proc

import (
	"fmt"
	"time"
)

type (
	Counts struct {
		Cpu        float64
		ReadBytes  uint64
		WriteBytes uint64
	}

	Memory struct {
		Resident uint64
		Virtual  uint64
	}

	Filedesc struct {
		Open  int64 // -1 if we don't know
		Limit uint64
	}

	// Tracker tracks processes and records metrics.
	Tracker struct {
		// Tracked holds the processes are being monitored.  Processes
		// may be blacklisted such that they no longer get tracked by
		// setting their value in the Tracked map to nil.
		Tracked map[ProcId]*TrackedProc
		// ProcIds is a map from pid to ProcId.  This is a convenience
		// to allow finding the Tracked entry of a parent process.
		ProcIds map[int]ProcId
	}

	// TrackedProc accumulates metrics for a process, as well as
	// remembering an optional GroupName tag associated with it.
	TrackedProc struct {
		// lastUpdate is used internally during the update cycle to find which procs have exited
		lastUpdate time.Time
		// info is the most recently obtained info for this proc
		info ProcInfo
		// accum is the total CPU and IO accrued since we started tracking this proc
		accum Counts
		// lastaccum is the CPU and IO accrued in the last Update()
		lastaccum Counts
		// GroupName is an optional tag for this proc.
		GroupName string
	}

	trackedStats struct {
		aggregate, latest Counts
		Memory
		Filedesc
		start time.Time
	}

	collectErrors struct {
		// readErrors is incremented every time GetMetrics() returns an error.
		Read int
		// permissionErrors is incremented every time we're unable to collect
		// some metrics (e.g. I/O) for a tracked proc.
		Permission int
	}
)

func (tp *TrackedProc) GetName() string {
	return tp.info.Name
}

func (tp *TrackedProc) GetCmdLine() []string {
	return tp.info.Cmdline
}

func (tp *TrackedProc) GetStats() trackedStats {
	mem := Memory{Resident: tp.info.ResidentBytes, Virtual: tp.info.VirtualBytes}
	fd := Filedesc{Open: tp.info.OpenFDs, Limit: tp.info.MaxFDs}
	return trackedStats{
		aggregate: tp.accum,
		latest:    tp.lastaccum,
		Memory:    mem,
		Filedesc:  fd,
		start:     tp.info.StartTime,
	}
}

func NewTracker() *Tracker {
	return &Tracker{Tracked: make(map[ProcId]*TrackedProc), ProcIds: make(map[int]ProcId)}
}

func (t *Tracker) Track(groupName string, idinfo ProcIdInfo) {
	info := ProcInfo{idinfo.ProcStatic, idinfo.ProcMetrics}
	t.Tracked[idinfo.ProcId] = &TrackedProc{GroupName: groupName, info: info}
}

func (t *Tracker) Ignore(id ProcId) {
	t.Tracked[id] = nil
}

// Scan procs and update metrics for those which are tracked.  Processes that have gone
// away get removed from the Tracked map.  New processes are returned, along with the count
// of permission errors.
func (t *Tracker) Update(procs ProcIter) ([]ProcIdInfo, collectErrors, error) {
	now := time.Now()
	var newProcs []ProcIdInfo
	var colErrs collectErrors

	for procs.Next() {
		procId, err := procs.GetProcId()
		if err != nil {
			continue
		}

		last, known := t.Tracked[procId]

		// Are we ignoring this proc?
		if known && last == nil {
			continue
		}

		metrics, err := procs.GetMetrics()
		if err != nil {
			// This usually happens due to the proc having exited, i.e.
			// we lost the race.  We don't count that as an error.
			if err != ErrProcNotExist {
				colErrs.Read++
			}
			continue
		}

		if known {
			// newcounts: resource consumption since last cycle
			newcounts := Counts{Cpu: metrics.CpuTime - last.info.CpuTime}
			// Counts are also used in Grouper, to track aggregate usage
			// across groups.  It would be nice to expose that we weren't
			// able to get some metrics (e.g. due to permissions) but not nice
{'fraction_non_alphanumeric': 0.08852901484480431, 'fraction_numerical': 0.004048582995951417, 'mean_word_length': 3.301218804410911, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7412411', 'n_tokens_mistral': 2593, 'n_tokens_neox': 2333, 'n_words': 888}