<commit_before>package web

import (
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/StackExchange/tsaf/expr"
	"github.com/StackExchange/tsaf/third_party/github.com/MiniProfiler/go/miniprofiler"
	"github.com/StackExchange/tsaf/third_party/github.com/StackExchange/scollector/opentsdb"
)

// Graph takes an OpenTSDB request data structure and queries OpenTSDB. Use the
// json parameter to pass JSON. Use the b64 parameter to pass base64-encoded
// JSON.
func Graph(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
	j := []byte(r.FormValue("json"))
	if bs := r.FormValue("b64"); bs != "" {
		b, err := base64.URLEncoding.DecodeString(bs)
		if err != nil {
			return nil, err
		}
		j = b
	}
	if len(j) == 0 {
		return nil, fmt.Errorf("either json or b64 required")
	}
	oreq, err := opentsdb.RequestFromJSON(j)
	if err != nil {
		return nil, err
	}
	ads_v := r.FormValue("autods")
	if ads_v != "" {
		ads_i, err := strconv.ParseInt(ads_v, 10, 64)
		if err != nil {
			return nil, err
		}
		if err := oreq.AutoDownsample(ads_i); err != nil {
			return nil, err
		}
	}
	for _, q := range oreq.Queries {
		if err := expr.ExpandSearch(q); err != nil {
			return nil, err
		}
	}
	if _, present := r.Form["png"]; present {
		u := url.URL{
			Scheme:   "http",
			Host:     schedule.Conf.TsdbHost,
			Path:     "/q",
			RawQuery: oreq.String() + "&png",
		}
		resp, err := http.Get(u.String())
		if err != nil {
			return nil, err
		}
		for k, v := range resp.Header {
			w.Header()[k] = v
		}
		w.WriteHeader(resp.StatusCode)
		_, err = io.Copy(w, resp.Body)
		return nil, err
	}
	var tr opentsdb.ResponseSet
	q, _ := url.QueryUnescape(oreq.String())
	t.StepCustomTiming("tsdb", "query", q, func() {
		tr, err = oreq.Query(schedule.Conf.TsdbHost)
	})
	if err != nil {
		return nil, err
	}
	chart, err := rickchart(tr)
	if err != nil {
		return nil, err
	}
	return struct {
		Queries []string
		Series  []*RickSeries
	}{
		QFromR(oreq),
		chart,
	}, nil
}

func QFromR(req *opentsdb.Request) []string {
	queries := make([]string, len(req.Queries))
	var start, end string
	if s, ok := req.Start.(string); ok {
		start = strings.TrimSuffix(s, "-ago")
	}
	if s, ok := req.End.(string); ok {
		end = strings.TrimSuffix(s, "-ago")
	}
	for i, q := range req.Queries {
		queries[i] = fmt.Sprintf(`q("%v", "%v", "%v")`, q, start, end)
	}
	return queries
}

var q_re = regexp.MustCompile(`"([^"]+)"\s*,\s*"([^"]+)"`)
var r_re = regexp.MustCompile(`^(.*?")[^"]+(".*)`)

func ExprGraph(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
	q := r.FormValue("q")
	qs := q_re.FindStringSubmatch(q)
	if qs == nil {
		return nil, errors.New("couldn't parse out query")
	}
	pq, err := opentsdb.ParseQuery(qs[1])
	oreq := opentsdb.Request{
		Start:   qs[2] + "-ago",
		Queries: []*opentsdb.Query{pq},
	}
	ads_v := r.FormValue("autods")
	if ads_v != "" {
		ads_i, err := strconv.ParseInt(ads_v, 10, 64)
		if err != nil {
			return nil, err
		}
		if err := oreq.AutoDownsample(ads_i); err != nil {
			return nil, err
		}
	}
	if err != nil {
		return nil, err
	}
	n_qs := r_re.ReplaceAllString(q, "${1}"+oreq.Queries[0].String()+"${2}")
	e, err := expr.New(n_qs)
	if err != nil {
		return nil, err
	}
	res, _, err := e.Execute(opentsdb.Host(schedule.Conf.TsdbHost), t)
	if err != nil {
		return nil, err
	}
	return rickexpr(res, n_qs)
}

func rickexpr(r []*expr.Result, q string) ([]*RickSeries, error) {
	series := make([]*RickSeries, 0)
	for _, res := range r {
		dps := make([]RickDP, 0)
		var rv expr.Series
		var ok bool
		if rv, ok = res.Value.(expr.Series); !ok {
			return series, errors.New("expr must return a series")
		}
		for k, v := range rv {
			ki, err := strconv.ParseInt(k, 10, 64)
			if err != nil {
				return nil, err
			}
			dps = append(dps, RickDP{
				X: ki,
				Y: v,
			})
		}
		if len(dps) > 0 {
			sort.Sort(ByX(dps))
			name := q
			var id []string
			for k, v := range res.Group {
				id = append(id, fmt.Sprintf("%v=%v", k, v))
			}
			if len(id) > 0 {
				name = fmt.Sprintf("%s{%s}", name, strings.Join(id, ","))
			}
			series = append(series, &RickSeries{
				Name: name,
				Data: dps,
			})
		}
	}
	return series, nil
}

func rickchart(r opentsdb.ResponseSet) ([]*RickSeries, error) {
	var series []*RickSeries
	for _, resp := range r {
		dps := make([]RickDP, 0)
		for k, v := range resp.DPS {
			ki, err := strconv.ParseInt(k, 10, 64)
			if err != nil {
				return nil, err
			}
			dps = append(dps, RickDP{
				X: ki,
				Y: v,
			})
		}
		if len(dps) > 0 {
			sort.Sort(ByX(dps))
			name := resp.Metric
			var id []string
			for k, v := range resp.Tags {
				id = append(id, fmt.Sprintf("%v=%v", k, v))
			}
			if len(id) > 0 {
				name = fmt.Sprintf("%s{%s}", name, strings.Join(id, ","))
			}
			series = append(series, &RickSeries{
				Name: name,
				Data: dps,
			})
		}
	}
	return series, nil
}

type RickSeries struct {
	Name string   `json:"name"`
	Data []RickDP `json:"data"`
}

type RickDP struct {
	X int64          `json:"x"`
	Y opentsdb.Point `json:"y"`
}

type ByX []RickDP

func (a ByX) Len() int           { return len(a) }
func (a ByX) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByX) Less(i, j int) bool { return a[i].X < a[j].X }
<commit_msg>Set q params to "" if time isn't relative<commit_after>package web

import (
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/StackExchange/tsaf/expr"
	"github.com/StackExchange/tsaf/third_party/github.com/MiniProfiler/go/miniprofiler"
	"github.com/StackExchange/tsaf/third_party/github.com/StackExchange/scollector/opentsdb"
)

// Graph takes an OpenTSDB request data structure and queries OpenTSDB. Use the
// json parameter to pass JSON. Use the b64 parameter to pass base64-encoded
// JSON.
func Graph(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
	j := []byte(r.FormValue("json"))
	if bs := r.FormValue("b64"); bs != "" {
		b, err := base64.URLEncoding.DecodeString(bs)
		if err != nil {
			return nil, err
		}
		j = b
	}
	if len(j) == 0 {
		return nil, fmt.Errorf("either json or b64 required")
	}
	oreq, err := opentsdb.RequestFromJSON(j)
	if err != nil {
		return nil, err
	}
	ads_v := r.FormValue("autods")
	if ads_v != "" {
		ads_i, err := strconv.ParseInt(ads_v, 10, 64)
		if err != nil {
			return nil, err
		}
		if err := oreq.AutoDownsample(ads_i); err != nil {
			return nil, err
		}
	}
	for _, q := range oreq.Queries {
		if err := expr.ExpandSearch(q); err != nil {
			return nil, err
		}
	}
	if _, present := r.Form["png"]; present {
		u := url.URL{
			Scheme:   "http",
			Host:     schedule.Conf.TsdbHost,
			Path:     "/q",
			RawQuery: oreq.String() + "&png",
		}
		resp, err := http.Get(u.String())
		if err != nil {
			return nil, err
		}
		for k, v := range resp.Header {
			w.Header()[k] = v
		}
		w.WriteHeader(resp.StatusCode)
		_, err = io.Copy(w, resp.Body)
		return nil, err
	}
	var tr opentsdb.ResponseSet
	q, _ := url.QueryUnescape(oreq.String())
	t.StepCustomTiming("tsdb", "query", q, func() {
		tr, err = oreq.Query(schedule.Conf.TsdbHost)
	})
	if err != nil {
		return nil, err
	}
	chart, err := rickchart(tr)
	if err != nil {
		return nil, err
	}
	return struct {
		Queries []string
		Series  []*RickSeries
	}{
		QFromR(oreq),
		chart,
	}, nil
}

func QFromR(req *opentsdb.Request) []string {
	queries := make([]string, len(req.Queries))
	var start, end string
	if s, ok := req.Start.(string); ok && strings.Contains(s, "-ago") {
		start = strings.TrimSuffix(s, "-ago")
	}
	if s, ok := req.End.(string); ok && strings.Contains(s, "-ago") {
		end = strings.TrimSuffix(s, "-ago")
	}
	for i, q := range req.Queries {
		queries[i] = fmt.Sprintf(`q("%v", "%v", "%v")`, q, start, end)
	}
	return queries
}

var q_re = regexp.MustCompile(`"([^"]+)"\s*,\s*"([^"]+)"`)
var r_re = regexp.MustCompile(`^(.*?")[^"]+(".*)`)

func ExprGraph(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
	q := r.FormValue("q")
	qs := q_re.FindStringSubmatch(q)
	if qs == nil {
		return nil, errors.New("couldn't parse out query")
	}
	pq, err := opentsdb.ParseQuery(qs[1])
	oreq := opentsdb.Request{
		Start:   qs[2] + "-ago",
		Queries: []*opentsdb.Query{pq},
	}
	ads_v := r.FormValue("autods")
	if ads_v != "" {
		ads_i, err := strconv.ParseInt(ads_v, 10, 64)
		if err != nil {
			return nil, err
		}
		if err := oreq.AutoDownsample(ads_i); err != nil {
			return nil, err
		}
	}
	if err != nil {
		return nil, err
	}
	n_qs := r_re.ReplaceAllString(q, "${1}"+oreq.Queries[0].String()+"${2}")
	e, err := expr.New(n_qs)
	if err != nil {
		return nil, err
	}
	res, _, err := e.Execute(opentsdb.Host(schedule.Conf.TsdbHost), t)
	if err != nil {
		return nil, err
	}
	return rickexpr(res, n_qs)
}

func rickexpr(r []*expr.Result, q string) ([]*RickSeries, error) {
	series := make([]*RickSeries, 0)
	for _, res := range r {
		dps := make([]RickDP, 0)
		var rv expr.Series
		var ok bool
		if rv, ok = res.Value.(expr.Series); !ok {
			return series, errors.New("expr must return a series")
		}
		for k, v := range rv {
			ki, err := strconv.ParseInt(k, 10, 64)
			if err != nil {
				return nil, err
			}
			dps = append(dps, RickDP{
				X: ki,
				Y: v,
			})
		}
		if len(dps) > 0 {
			sort.Sort(ByX(dps))
			name := q
			var id []string
			for k, v := range res.Group {
				id = append(id, fmt.Sprintf("%v=%v", k, v))
			}
			if len(id) > 0 {
				name = fmt.Sprintf("%s{%s}", name, strings.Join(id, ","))
			}
			series = append(series, &RickSeries{
				Name: name,
				Data: dps,
			})
		}
	}
	return series, nil
}

func rickchart(r opentsdb.ResponseSet) ([]*RickSeries, error) {
	var series []*RickSeries
	for _, resp := range r {
		dps := make([]RickDP, 0)
		for k, v := range resp.DPS {
			ki, err := strconv.ParseInt(k, 10, 64)
			if err != nil {
				return nil, err
			}
			dps = append(dps, RickDP{
				X: ki,
				Y: v,
			})
		}
		if len(dps) > 0 {
			sort.Sort(ByX(dps))
			name := resp.Metric
			var id []string
			for k, v := range resp.Tags {
				id = append(id, fmt.Sprintf("%v=%v", k, v))
			}
			if len(id) > 0 {
				name = fmt.Sprintf("%s{%s}", name, strings.Join(id, ","))
			}
			series = append(series, &RickSeries{
				Name: name,
				Data: dps,
			})
		}
	}
	return series, nil
}

type RickSeries struct {
	Name string   `json:"name"`
	Data []RickDP `json:"data"`
}

type RickDP struct {
	X int64          `json:"x"`
	Y opentsdb.Point `json:"y"`
}

type ByX []RickDP

func (a ByX) Len() int           { return len(a) }
func (a ByX) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByX) Less(i, j int) bool { return a[i].X < a[j].X }

{'fraction_non_alphanumeric': 0.18448500651890481, 'fraction_numerical': 0.008008940212330043, 'mean_word_length': 3.057045712126936, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '449992', 'n_tokens_mistral': 4855, 'n_tokens_neox': 4355, 'n_words': 1099}