<commit_before>package core

import (
	"bytes"
	// "encoding/csv"
	"errors"
	"fmt"
	"github.com/gwenn/yacr"
	"io"
	"net"
	"net/http"
	"net/url"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
)

const (
	rowLength = 62
)

// Exporter collects HAProxy stats from the given URI and exports them as
// warp10 metrics package.
type Exporter struct {
	URI   string
	mutex sync.RWMutex
	fetch func() (io.ReadCloser, error)

	metrics   map[int]string
	sensision bytes.Buffer
	labels    string
}

// NewExporter returns an initialized Exporter.
func NewExporter(uri string, timeout time.Duration, labels map[string]string, metrics []string) (*Exporter, error) {
	u, err := url.Parse(uri)
	if err != nil {
		return nil, err
	}

	var fetch func() (io.ReadCloser, error)
	switch u.Scheme {
	case "http", "https", "file":
		fetch = fetchHTTP(uri, timeout)
	case "unix":
		fetch = fetchUnix(u, timeout)
	default:
		return nil, fmt.Errorf("unsupported scheme: %q", u.Scheme)
	}

	e := &Exporter{
		URI:   uri,
		fetch: fetch,
		metrics: map[int]string{
			// pxname
			// svname
			2:  "qcur",
			3:  "qmax",
			4:  "scur",
			5:  "smax",
			6:  "slim",
			7:  "stot",
			8:  "bin",
			9:  "bout",
			10: "dreq",
			11: "dresp",
			12: "ereq",
			13: "econ",
			14: "eresp",
			15: "wretr",
			16: "wredis",
			17: "status",
			18: "weight",
			19: "act",
			20: "bck",
			21: "chkfail",
			22: "chkdown",
			23: "lastchg",
			24: "downtime",
			25: "qlimit",
			26: "pid",
			27: "iid",
			28: "sid",
			29: "throttle",
			30: "lbtot",
			31: "tracked",
			// type
			33: "current_session_rate",
			34: "limit_session_rate",
			35: "max_session_rate",
			// check_status
			// check_code
			38: "check_duration",
			39: "hrsp_1xx",
			40: "hrsp_2xx",
			41: "hrsp_3xx",
			42: "hrsp_4xx",
			43: "hrsp_5xx",
			44: "hrsp_other",
			// hanafail
			46: "req_rate",
			47: "req_rate_max",
			48: "req_tot",
			49: "cli_abrt",
			50: "srv_abrt",
			51: "comp_in",
			52: "comp_out",
			53: "comp_byp",

			54: "comp_rsp",
			55: "lastsess",
			// last_chk
			// last_agt
			58: "qtime",
			59: "ctime",
			60: "rtime",
			61: "ttime",
		},
	}

	// filter
	if len(metrics) > 0 {
		for i := range e.metrics {
			found := false
			for m := range metrics {
				if e.metrics[i] == metrics[m] {

					found = true
					break
				}
			}

			if !found {
				delete(e.metrics, i)
			}<commit_msg>feat: add check_status, check_code et last_chk<commit_after>package core

import (
	"bytes"
	// "encoding/csv"
	"errors"
	"fmt"
	"github.com/gwenn/yacr"
	"io"
	"net"
	"net/http"
	"net/url"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
)

const (
	rowLength = 62
)

// Exporter collects HAProxy stats from the given URI and exports them as
// warp10 metrics package.
type Exporter struct {
	URI   string
	mutex sync.RWMutex
	fetch func() (io.ReadCloser, error)

	metrics   map[int]string
	sensision bytes.Buffer
	labels    string
}

// NewExporter returns an initialized Exporter.
func NewExporter(uri string, timeout time.Duration, labels map[string]string, metrics []string) (*Exporter, error) {
	u, err := url.Parse(uri)
	if err != nil {
		return nil, err
	}

	var fetch func() (io.ReadCloser, error)
	switch u.Scheme {
	case "http", "https", "file":
		fetch = fetchHTTP(uri, timeout)
	case "unix":
		fetch = fetchUnix(u, timeout)
	default:
		return nil, fmt.Errorf("unsupported scheme: %q", u.Scheme)
	}

	e := &Exporter{
		URI:   uri,
		fetch: fetch,
		metrics: map[int]string{
			// pxname
			// svname
			2:  "qcur",
			3:  "qmax",
			4:  "scur",
			5:  "smax",
			6:  "slim",
			7:  "stot",
			8:  "bin",
			9:  "bout",
			10: "dreq",
			11: "dresp",
			12: "ereq",
			13: "econ",
			14: "eresp",
			15: "wretr",
			16: "wredis",
			17: "status",
			18: "weight",
			19: "act",
			20: "bck",
			21: "chkfail",
			22: "chkdown",
			23: "lastchg",
			24: "downtime",
			25: "qlimit",
			26: "pid",
			27: "iid",
			28: "sid",
			29: "throttle",
			30: "lbtot",
			31: "tracked",
			// type
			33: "current_session_rate",
			34: "limit_session_rate",
			35: "max_session_rate",
			36: "check_status",
			37: "check_code",
			38: "check_duration",
			39: "hrsp_1xx",
			40: "hrsp_2xx",
			41: "hrsp_3xx",
			42: "hrsp_4xx",
			43: "hrsp_5xx",
			44: "hrsp_other",
			// hanafail
			46: "req_rate",
			47: "req_rate_max",
			48: "req_tot",
			49: "cli_abrt",
			50: "srv_abrt",
			51: "comp_in",
			52: "comp_out",
			53: "comp_byp",

			54: "comp_rsp",
			55: "lastsess",
			56: "last_chk",
			// last_agt
			58: "qtime",
			59: "ctime",
			60: "rtime",
			61: "ttime",
		},
	}

	// filter
	if len(metrics) > 0 {
		for i := range e.metrics {
			found := false
			for m := range metrics {
				if e.metrics[i] == metrics[m] {

					found = true
					break
				}
			}

			if !found {
				delete(e.metrics, i)
			}
{'fraction_non_alphanumeric': 0.17240659570027134, 'fraction_numerical': 0.04717178042162388, 'mean_word_length': 2.708978328173375, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6369129', 'n_tokens_mistral': 2482, 'n_tokens_neox': 2046, 'n_words': 508}