<commit_before>package collectors

import (
	"bytes"
	"encoding/base64"
	"encoding/gob"
	"fmt"
	"net/http"
	"strconv"
	"time"

	analytics "bosun.org/_third_party/google.golang.org/api/analytics/v3"

	"bosun.org/_third_party/golang.org/x/net/context"
	"bosun.org/_third_party/golang.org/x/oauth2"
	"bosun.org/_third_party/golang.org/x/oauth2/google"
	"bosun.org/cmd/scollector/conf"
	"bosun.org/metadata"
	"bosun.org/opentsdb"
)

const descActiveUsers = "Number of unique users actively visiting the site."

func init() {

	registerInit(func(c *conf.Conf) {
		for _, g := range c.GoogleAnalytics {
			collectors = append(collectors, &IntervalCollector{
				F: func() (opentsdb.MultiDataPoint, error) {
					return c_google_analytics(g.ClientID, g.Secret, g.Token, g.Sites)
				},
				name:     "c_google_analytics",
				Interval: time.Minute * 1,
			})
		}
	})
}

func c_google_analytics(clientid string, secret string, tokenstr string, sites []conf.GoogleAnalyticsSite) (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint

	c, err := analyticsClient(clientid, secret, tokenstr)
	if err != nil {
		return nil, err
	}
	svc, err := analytics.New(c)
	if err != nil {
		return nil, err
	}

	for _, site := range sites {
		getPageviews(&md, svc, site)
		if site.Detailed {
			getActiveUsers(&md, svc, site)
			getActiveUsersByDimension(&md, svc, site, "browser")
			getActiveUsersByDimension(&md, svc, site, "trafficType")
			getActiveUsersByDimension(&md, svc, site, "deviceCategory")
			getActiveUsersByDimension(&md, svc, site, "operatingSystem")
		}
	}

	return md, err
}

func getActiveUsersByDimension(md *opentsdb.MultiDataPoint, svc *analytics.Service, site conf.GoogleAnalyticsSite, dimension string) error {
	call := svc.Data.Realtime.Get("ga:"+site.Profile, "rt:activeusers").Dimensions("rt:" + dimension)
	data, err := call.Do()
	if err != nil {
		return err
	}
	tags := opentsdb.TagSet{"site": site.Name}
	for _, row := range data.Rows {
		// key will always be an string of the dimension we care about.
		// For example, 'Chrome' would be a key for the 'browser' dimension.
		key := row[0]
		value, err := strconv.Atoi(row[1])
		if err != nil {
			return fmt.Errorf("Error parsing GA data: %s", err)
		}

		Add(md, "google.analytics.realtime.activeusers.by_"+dimension, value, opentsdb.TagSet{dimension: key}.Merge(tags), metadata.Gauge, metadata.ActiveUsers, descActiveUsers)
	}
	return nil
}

func getActiveUsers(md *opentsdb.MultiDataPoint, svc *analytics.Service, site conf.GoogleAnalyticsSite) error {
	call := svc.Data.Realtime.Get("ga:"+site.Profile, "rt:activeusers")
	data, err := call.Do()
	if err != nil {
		return err
	}
	tags := opentsdb.TagSet{"site": site.Name}
<commit_msg>c_google_analytics: Actually gather GA errors.<commit_after>package collectors

import (
	"bytes"
	"encoding/base64"
	"encoding/gob"
	"fmt"
	"net/http"
	"strconv"
	"time"

	analytics "bosun.org/_third_party/google.golang.org/api/analytics/v3"

	"bosun.org/_third_party/golang.org/x/net/context"
	"bosun.org/_third_party/golang.org/x/oauth2"
	"bosun.org/_third_party/golang.org/x/oauth2/google"
	"bosun.org/cmd/scollector/conf"
	"bosun.org/metadata"
	"bosun.org/opentsdb"
)

const descActiveUsers = "Number of unique users actively visiting the site."

type multiError []error

func (m multiError) Error() string {
	var fullErr string
	for _, err := range m {
		fullErr = fmt.Sprintf("%s\n%s", fullErr, err)
	}
	return fullErr
}

func init() {
	registerInit(func(c *conf.Conf) {
		for _, g := range c.GoogleAnalytics {
			collectors = append(collectors, &IntervalCollector{
				F: func() (opentsdb.MultiDataPoint, error) {
					return c_google_analytics(g.ClientID, g.Secret, g.Token, g.Sites)
				},
				name:     "c_google_analytics",
				Interval: time.Minute * 1,
			})
		}
	})
}

func c_google_analytics(clientid string, secret string, tokenstr string, sites []conf.GoogleAnalyticsSite) (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	var mErr multiError

	c, err := analyticsClient(clientid, secret, tokenstr)
	if err != nil {
		return nil, err
	}
	svc, err := analytics.New(c)
	if err != nil {
		return nil, err
	}

	dimensions := []string{"browser", "trafficType", "deviceCategory", "operatingSystem"}
	for _, site := range sites {
		getPageviews(&md, svc, site)
		if site.Detailed {
			if err = getActiveUsers(&md, svc, site); err != nil {
				mErr = append(mErr, err)
			}
			for _, dimension := range dimensions {
				if err = getActiveUsersByDimension(&md, svc, site, dimension); err != nil {
					mErr = append(mErr, err)
				}
			}
		}
	}

	return md, mErr
}

func getActiveUsersByDimension(md *opentsdb.MultiDataPoint, svc *analytics.Service, site conf.GoogleAnalyticsSite, dimension string) error {
	call := svc.Data.Realtime.Get("ga:"+site.Profile, "rt:activeusers").Dimensions("rt:" + dimension)
	data, err := call.Do()
	if err != nil {
		return err
	}
	tags := opentsdb.TagSet{"site": site.Name}
	for _, row := range data.Rows {
		// key will always be an string of the dimension we care about.
		// For example, 'Chrome' would be a key for the 'browser' dimension.
		key := row[0]
		value, err := strconv.Atoi(row[1])
		if err != nil {
			return fmt.Errorf("Error parsing GA data: %s", err)
		}

		Add(md, "google.analytics.realtime.activeusers.by_"+dimension, value, opentsdb.TagSet{dimension: key}.Merge(tags), metadata.Gauge, metadata.ActiveUsers, descActiveUsers)
	}
	return nil
}

func getActiveUsers(md *opentsdb.MultiDataPoint, svc *analytics.Service, site conf.GoogleAnalyticsSite) error {
	call := svc.Data.Realtime.Get("ga:"+site.Profile, "rt:activeusers")
	data, err := call.Do()
	if err != nil {
		return err
	}
	tags := opentsdb.TagSet{"site": site.Name}

{'fraction_non_alphanumeric': 0.14058401818499738, 'fraction_numerical': 0.002797691904179052, 'mean_word_length': 4.257352941176471, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7240154', 'n_tokens_mistral': 2216, 'n_tokens_neox': 2061, 'n_words': 473}