<commit_before>package collectors

import (
	"fmt"
	"strconv"
	"strings"

	"bosun.org/_third_party/golang.org/x/sys/unix"
	"bosun.org/metadata"
	"bosun.org/opentsdb"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_procstats_linux})
}

var CPU_FIELDS = []string{
	"user",
	"nice",
	"system",
	"idle",
	"iowait",
	"irq",
	"softirq",
	"steal",
	"guest",
	"guest_nice",
}

func c_procstats_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	var Error error
	var sys unix.Sysinfo_t

	unix.Sysinfo(&sys)

	Add(&md, "linux.uptime_total", sys.Uptime, nil, metadata.Gauge, metadata.Second, osSystemUptimeDesc)
	Add(&md, osSystemUptime, sys.Uptime, nil, metadata.Gauge, metadata.Second, osSystemUptimeDesc)
	if err := readLine("/proc/meminfo", func(s string) error {
		s = strings.TrimSuffix(s, " kB")
		m := strings.Split(s, ":")
		if m == nil {
			return nil
		}
		m[1] = strings.TrimSpace(m[1])
		Add(&md, "linux.mem."+strings.ToLower(m[0]), m[1], nil, metadata.Gauge, metadata.KBytes, "")
		return nil
	}); err != nil {
		Error = err
	}
	Add(&md, osMemTotal, sys.Totalram*uint64(sys.Unit), nil, metadata.Gauge, metadata.Bytes, osMemTotalDesc)
	Add(&md, osMemFree, sys.Freeram*uint64(sys.Unit), nil, metadata.Gauge, metadata.Bytes, osMemFreeDesc)
	Add(&md, osMemUsed, (sys.Totalram-sys.Freeram)*uint64(sys.Unit), nil, metadata.Gauge, metadata.Bytes, osMemUsedDesc)
	Add(&md, "linux.loadavg.1_min", sys.Loads[0], nil, metadata.Gauge, metadata.Load, "")
	Add(&md, "linux.loadavg.5_min", sys.Loads[1], nil, metadata.Gauge, metadata.Load, "")
	Add(&md, "linux.loadavg.15_min", sys.Loads[2], nil, metadata.Gauge, metadata.Load, "")
	Add(&md, "linux.loadavg.total_threads", sys.Procs, nil, metadata.Gauge, metadata.Process, "")
	if sys.Totalram != 0 {
		Add(&md, osMemPctFree, (sys.Freeram)/(sys.Totalram)*100, nil, metadata.Gauge, metadata.Pct, osMemFreeDesc)
	}
	if err := readLine("/proc/vmstat", func(s string) error {
		m := strings.Split(s, " ")
		if m == nil {
			return nil
		}
		switch m[0] {
		case "pgpgin", "pgpgout", "pswpin", "pswpout":
			switch {
			case strings.HasSuffix(m[0], "in"):
				Add(&md, "linux.mem."+strings.TrimSuffix(m[0], "in"), m[1], opentsdb.TagSet{"direction": "in"}, metadata.Counter, metadata.Page, "")
			case strings.HasSuffix(m[0], "out"):
				Add(&md, "linux.mem."+strings.TrimSuffix(m[0], "out"), m[1], opentsdb.TagSet{"direction": "out"}, metadata.Counter, metadata.Page, "")
			}
		case "pgfault", "pgmajfault":
			Add(&md, "linux.mem."+m[0], m[1], nil, metadata.Counter, metadata.Page, "")
		default:
			Add(&md, "linux.mem."+m[0], m[1], nil, metadata.Counter, metadata.None, "")
		}
		return nil
	}); err != nil {
		Error = err
	}
	num_cores := 0
	var t_util int
	cpu_stat_desc := map[string]string{
		"user":       "Normal processes executing in user mode.",
		"nice":       "Niced processes executing in user mode.",
		"system":     "Processes executing in kernel mode.",
		"idle":       "Twiddling thumbs.",
		"iowait":     "Waiting for I/O to complete.",
		"irq":        "Servicing interrupts.",
		"softirq":    "Servicing soft irqs.",
		"steal":      "Involuntary wait.",
		"guest":      "Running a guest vm.",
		"guest_nice": "Running a niced guest vm.",
	}
	if err := readLine("/proc/stat", func(s string) error {
		m := strings.Split(s, " ")
		if m == nil {
			return nil
		}
		switch {
		case strings.HasPrefix(m[0], "cpu"):
			metric_percpu := ""
			tag_cpu := strings.TrimPrefix(m[0], "cpu")
			if tag_cpu != "" {
				num_cores++
				metric_percpu = ".percpu"
			}
			fields := strings.Fields(m[1])
			for i, value := range fields {
				if i >= len(CPU_FIELDS) {
					break
				}
				tags := opentsdb.TagSet{
					"type": CPU_FIELDS[i],
				}
				if tag_cpu != "" {
					tags["cpu"] = tag_cpu
				}
				Add(&md, "linux.cpu.percpu", value, tags, metadata.Counter, metadata.CHz, cpu_stat_desc[CPU_FIELDS[i]])
			}
			if metric_percpu == "" {
				if len(fields) < 3 {
					return nil
				}
				user, err := strconv.Atoi(fields[0])
				if err != nil {
					return nil
				}
				nice, err := strconv.Atoi(fields[1])
				if err != nil {
					return nil
				}
				system, err := strconv.Atoi(fields[2])
				if err != nil {
					return nil
				}
				t_util = user + nice + system
			}
		case m[0] == "intr":
			Add(&md, "linux.intr", strings.Fields(m[1])[0], nil, metadata.Counter, metadata.Interupt, "")
		case m[0] == "ctxt":
			Add(&md, "linux.ctxt", m[1], nil, metadata.Counter, metadata.ContextSwitch, "")
		case m[0] == "processes":
			Add(&md, "linux.processes", m[1], nil, metadata.Counter, metadata.Process,
				"The number  of processes and threads created, which includes (but  is not limited  to) those  created by  calls to the  fork() and clone() system calls.")
		case m[0] == "procs_blocked":
			Add(&md, "linux.procs_blocked", m[1], nil, metadata.Gauge, metadata.Process, "The  number of  processes currently blocked, waiting for I/O to complete.")
		}
		return nil
	}); err != nil {
		Error = err
	}<commit_msg>cmd/scollector: fixed issues with some cpu statistics missing in linux.<commit_after>package collectors

import (
	"fmt"
	"strconv"
	"strings"

	"bosun.org/_third_party/golang.org/x/sys/unix"
	"bosun.org/metadata"
	"bosun.org/opentsdb"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_procstats_linux})
}

var CPU_FIELDS = []string{
	"user",
	"nice",
	"system",
	"idle",
	"iowait",
	"irq",
	"softirq",
	"steal",
	"guest",
	"guest_nice",
}

func c_procstats_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	var Error error
	var sys unix.Sysinfo_t

	unix.Sysinfo(&sys)

	Add(&md, "linux.uptime_total", sys.Uptime, nil, metadata.Gauge, metadata.Second, osSystemUptimeDesc)
	Add(&md, osSystemUptime, sys.Uptime, nil, metadata.Gauge, metadata.Second, osSystemUptimeDesc)
	if err := readLine("/proc/meminfo", func(s string) error {
		s = strings.TrimSuffix(s, " kB")
		m := strings.Split(s, ":")
		if m == nil {
			return nil
		}
		m[1] = strings.TrimSpace(m[1])
		Add(&md, "linux.mem."+strings.ToLower(m[0]), m[1], nil, metadata.Gauge, metadata.KBytes, "")
		return nil
	}); err != nil {
		Error = err
	}
	Add(&md, osMemTotal, sys.Totalram*uint64(sys.Unit), nil, metadata.Gauge, metadata.Bytes, osMemTotalDesc)
	Add(&md, osMemFree, sys.Freeram*uint64(sys.Unit), nil, metadata.Gauge, metadata.Bytes, osMemFreeDesc)
	Add(&md, osMemUsed, (sys.Totalram-sys.Freeram)*uint64(sys.Unit), nil, metadata.Gauge, metadata.Bytes, osMemUsedDesc)
	Add(&md, "linux.loadavg.1_min", sys.Loads[0], nil, metadata.Gauge, metadata.Load, "")
	Add(&md, "linux.loadavg.5_min", sys.Loads[1], nil, metadata.Gauge, metadata.Load, "")
	Add(&md, "linux.loadavg.15_min", sys.Loads[2], nil, metadata.Gauge, metadata.Load, "")
	Add(&md, "linux.loadavg.total_threads", sys.Procs, nil, metadata.Gauge, metadata.Process, "")
	if sys.Totalram != 0 {
		Add(&md, osMemPctFree, (sys.Freeram)/(sys.Totalram)*100, nil, metadata.Gauge, metadata.Pct, osMemFreeDesc)
	}
	if err := readLine("/proc/vmstat", func(s string) error {
		m := strings.Split(s, " ")
		if m == nil {
			return nil
		}
		switch m[0] {
		case "pgpgin", "pgpgout", "pswpin", "pswpout":
			switch {
			case strings.HasSuffix(m[0], "in"):
				Add(&md, "linux.mem."+strings.TrimSuffix(m[0], "in"), m[1], opentsdb.TagSet{"direction": "in"}, metadata.Counter, metadata.Page, "")
			case strings.HasSuffix(m[0], "out"):
				Add(&md, "linux.mem."+strings.TrimSuffix(m[0], "out"), m[1], opentsdb.TagSet{"direction": "out"}, metadata.Counter, metadata.Page, "")
			}
		case "pgfault", "pgmajfault":
			Add(&md, "linux.mem."+m[0], m[1], nil, metadata.Counter, metadata.Page, "")
		default:
			Add(&md, "linux.mem."+m[0], m[1], nil, metadata.Counter, metadata.None, "")
		}
		return nil
	}); err != nil {
		Error = err
	}
	num_cores := 0
	var t_util int
	cpu_stat_desc := map[string]string{
		"user":       "Normal processes executing in user mode.",
		"nice":       "Niced processes executing in user mode.",
		"system":     "Processes executing in kernel mode.",
		"idle":       "Twiddling thumbs.",
		"iowait":     "Waiting for I/O to complete.",
		"irq":        "Servicing interrupts.",
		"softirq":    "Servicing soft irqs.",
		"steal":      "Involuntary wait.",
		"guest":      "Running a guest vm.",
		"guest_nice": "Running a niced guest vm.",
	}
	if err := readLine("/proc/stat", func(s string) error {
		m := strings.Split(s, " ")
		if m == nil {
			return nil
		}
		switch {
		case strings.HasPrefix(m[0], "cpu"):
			tag_cpu := strings.TrimPrefix(m[0], "cpu")
			if tag_cpu != "" {
				num_cores++
			}
			for i, value := range m[1:] {
				if i >= len(CPU_FIELDS) {
					break
				}
				tags := opentsdb.TagSet{
					"type": CPU_FIELDS[i],
				}
				if tag_cpu != "" {
					tags["cpu"] = tag_cpu
					Add(&md, "linux.cpu.percpu", value, tags, metadata.Counter, metadata.CHz, cpu_stat_desc[CPU_FIELDS[i]])
				} else {
					Add(&md, "linux.cpu", value, tags, metadata.Counter, metadata.CHz, cpu_stat_desc[CPU_FIELDS[i]])
				}
			}
			if tag_cpu == "" {
				if len(m[1:]) < 3 {
					return nil
				}
				user, err := strconv.Atoi(m[1])
				if err != nil {
					return nil
				}
				nice, err := strconv.Atoi(m[2])
				if err != nil {
					return nil
				}
				system, err := strconv.Atoi(m[3])
				if err != nil {
					return nil
				}
				t_util = user + nice + system
			}
		case m[0] == "intr":
			Add(&md, "linux.intr", m[1], nil, metadata.Counter, metadata.Interupt, "")
		case m[0] == "ctxt":
			Add(&md, "linux.ctxt", m[1], nil, metadata.Counter, metadata.ContextSwitch, "")
		case m[0] == "processes":
			Add(&md, "linux.processes", m[1], nil, metadata.Counter, metadata.Process,
				"The number  of processes and threads created, which includes (but  is not limited  to) those  created by  calls to the  fork() and clone() system calls.")
		case m[0] == "procs_blocked":
			Add(&md, "linux.procs_blocked", m[1], nil, metadata.Gauge, metadata.Process, "The  number of  processes currently blocked, waiting for I/O to complete.")
		}
		return nil
	}); err != nil {
		Error = err
	}
{'fraction_non_alphanumeric': 0.18014888337468982, 'fraction_numerical': 0.009727047146401985, 'mean_word_length': 3.6756380510440834, 'pattern_counts': {'":': 40, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '923655', 'n_tokens_mistral': 4244, 'n_tokens_neox': 3821, 'n_words': 868}