<commit_before>// +build darwin linux

package collectors

import (
	"fmt"
	"io/ioutil"
	"regexp"
	"strings"
	"time"

	"github.com/garyburd/redigo/redis"

	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/slog"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_redis_linux})
}

var FIELDS_REDIS = map[string]bool{
	"bgrewriteaof_in_progress":   true,
	"bgsave_in_progress":         true,
	"blocked_clients":            true,
	"changes_since_last_save":    true,
	"client_biggest_input_buf":   true,
	"client_longest_output_list": true,
	"connected_clients":          true,
	"connected_slaves":           true,
	"evicted_keys":               true,
	"expired_keys":               true,
	"hash_max_zipmap_entries":    true,
	"hash_max_zipmap_value":      true,
	"keyspace_hits":              true,
	"keyspace_misses":            true,
	"mem_fragmentation_ratio":    true,
	"pubsub_channels":            true,
	"pubsub_patterns":            true,
	"total_commands_processed":   true,
	"total_connections_received": true,
	"uptime_in_seconds":          true,
	"used_cpu_sys":               true,
	"used_cpu_user":              true,
	"used_memory":                true,
	"used_memory_rss":            true,
}

var (
	tcRE           = regexp.MustCompile(`^\s*#\s*scollector.(\w+)\s*=\s*(.+)$`)
	redisInstances map[string]string
)

func init() {
	update := func() {
		ri := make(map[string]string)
		oldRedis := false
		add := func(port, pid string) {
			cluster := fmt.Sprintf("port-%s", port)
			f, err := ioutil.ReadFile(fmt.Sprintf("/proc/%s/cmdline", pid))
			if err != nil {
				return
			}
			fsp := strings.Split(strings.Split(string(f), "\n")[0], "\u0000")
			if len(fsp) < 2 {
				return
			}
			cfg := fsp[len(fsp)-2]
			readLine(cfg, func(cfgline string) {
				result := tcRE.FindStringSubmatch(cfgline)
				if len(result) > 2 && strings.ToLower(result[0]) == "cluster" {
					cluster = strings.ToLower(result[1])
				}
			})
			ri[port] = cluster
		}
		readCommand(func(line string) {
			sp := strings.Fields(line)
			if len(sp) != 3 || !strings.HasSuffix(sp[1], "redis-server") {
				return
			}
			if !strings.Contains(sp[2], ":") {
				oldRedis = true
				return
			}
			pid := sp[0]
			port := strings.Split(sp[2], ":")[1]
			add(port, pid)
		}, "ps", "-e", "-o", "pid,args")
		if oldRedis {
			readCommand(func(line string) {
				if !strings.Contains(line, "redis-server") {
					return
				}
				sp := strings.Fields(line)
				if len(sp) < 7 || !strings.Contains(sp[3], ":") {
					return
				}
				pid := strings.Split(sp[6], "/")[0]
				port := strings.Split(sp[3], ":")[1]
				add(port, pid)
			}, "netstat", "-tnlp")
		}
		redisInstances = ri
	}
	update()
	go func() {
		for _ = range time.Tick(time.Minute * 5) {
			update()
		}
	}()
}

func c_redis_linux() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	for port, cluster := range redisInstances {
		c, err := redis.Dial("tcp", fmt.Sprintf(":%s", port))
		if err != nil {
			slog.Infoln(err)
			continue
		}
		defer c.Close()
		tags := opentsdb.TagSet{
			"cluster": cluster,
			"port":    port,
		}
		lines, err := c.Do("INFO")
		if err != nil {
			slog.Infoln(err)
			continue
		}
		_ = tags
		for _, line := range strings.Split(string(lines.([]uint8)), "\n") {
			line = strings.TrimSpace(line)
			sp := strings.Split(line, ":")
			if len(sp) < 2 || !FIELDS_REDIS[sp[0]] {
				continue
			}
			Add(&md, "redis."+sp[0], sp[1], tags)
		}
	}
	return md
}
<commit_msg>cmd/scollector: Ignore lacking redis config file<commit_after>// +build darwin linux

package collectors

import (
	"fmt"
	"io/ioutil"
	"regexp"
	"strings"
	"time"

	"github.com/garyburd/redigo/redis"

	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/slog"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_redis_linux})
}

var FIELDS_REDIS = map[string]bool{
	"bgrewriteaof_in_progress":   true,
	"bgsave_in_progress":         true,
	"blocked_clients":            true,
	"changes_since_last_save":    true,
	"client_biggest_input_buf":   true,
	"client_longest_output_list": true,
	"connected_clients":          true,
	"connected_slaves":           true,
	"evicted_keys":               true,
	"expired_keys":               true,
	"hash_max_zipmap_entries":    true,
	"hash_max_zipmap_value":      true,
	"keyspace_hits":              true,
	"keyspace_misses":            true,
	"mem_fragmentation_ratio":    true,
	"pubsub_channels":            true,
	"pubsub_patterns":            true,
	"total_commands_processed":   true,
	"total_connections_received": true,
	"uptime_in_seconds":          true,
	"used_cpu_sys":               true,
	"used_cpu_user":              true,
	"used_memory":                true,
	"used_memory_rss":            true,
}

var (
	tcRE           = regexp.MustCompile(`^\s*#\s*scollector.(\w+)\s*=\s*(.+)$`)
	redisInstances map[string]string
)

func init() {
	update := func() {
		ri := make(map[string]string)
		oldRedis := false
		add := func(port, pid string) {
			cluster := fmt.Sprintf("port-%s", port)
			defer func() {
				ri[port] = cluster
			}()
			f, err := ioutil.ReadFile(fmt.Sprintf("/proc/%s/cmdline", pid))
			if err != nil {
				return
			}
			fsp := strings.Split(strings.Split(string(f), "\n")[0], "\u0000")
			if len(fsp) < 2 {
				return
			}
			cfg := fsp[len(fsp)-2]
			readLine(cfg, func(cfgline string) {
				result := tcRE.FindStringSubmatch(cfgline)
				if len(result) > 2 && strings.ToLower(result[0]) == "cluster" {
					cluster = strings.ToLower(result[1])
				}
			})
		}
		readCommand(func(line string) {
			sp := strings.Fields(line)
			if len(sp) != 3 || !strings.HasSuffix(sp[1], "redis-server") {
				return
			}
			if !strings.Contains(sp[2], ":") {
				oldRedis = true
				return
			}
			pid := sp[0]
			port := strings.Split(sp[2], ":")[1]
			add(port, pid)
		}, "ps", "-e", "-o", "pid,args")
		if oldRedis {
			readCommand(func(line string) {
				if !strings.Contains(line, "redis-server") {
					return
				}
				sp := strings.Fields(line)
				if len(sp) < 7 || !strings.Contains(sp[3], ":") {
					return
				}
				pid := strings.Split(sp[6], "/")[0]
				port := strings.Split(sp[3], ":")[1]
				add(port, pid)
			}, "netstat", "-tnlp")
		}
		redisInstances = ri
	}
	update()
	go func() {
		for _ = range time.Tick(time.Minute * 5) {
			update()
		}
	}()
}

func c_redis_linux() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	for port, cluster := range redisInstances {
		c, err := redis.Dial("tcp", fmt.Sprintf(":%s", port))
		if err != nil {
			slog.Infoln(err)
			continue
		}
		defer c.Close()
		tags := opentsdb.TagSet{
			"cluster": cluster,
			"port":    port,
		}
		lines, err := c.Do("INFO")
		if err != nil {
			slog.Infoln(err)
			continue
		}
		_ = tags
		for _, line := range strings.Split(string(lines.([]uint8)), "\n") {
			line = strings.TrimSpace(line)
			sp := strings.Split(line, ":")
			if len(sp) < 2 || !FIELDS_REDIS[sp[0]] {
				continue
			}
			Add(&md, "redis."+sp[0], sp[1], tags)
		}
	}
	return md
}

{'fraction_non_alphanumeric': 0.17155820218346804, 'fraction_numerical': 0.007939883737416702, 'mean_word_length': 2.8673245614035086, 'pattern_counts': {'":': 64, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5614704', 'n_tokens_mistral': 3054, 'n_tokens_neox': 2709, 'n_words': 493}